diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig deleted file mode 100644 index fe40ba4bbb16ffcbcc2768af9a6f8a57600c1e4f..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/CodeGen.zig +++ /dev/null @@ -1,8494 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const build_options = @import("build_options"); - -const mem = std.mem; -const math = std.math; -const assert = std.debug.assert; -const Allocator = mem.Allocator; - -const Air = @import("../../Air.zig"); -const Mir = @import("Mir.zig"); -const Emit = @import("Emit.zig"); -const Type = @import("../../Type.zig"); -const Value = @import("../../Value.zig"); -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); -const Package = @import("../../Package.zig"); -const InternPool = @import("../../InternPool.zig"); -const Compilation = @import("../../Compilation.zig"); -const target_util = @import("../../target.zig"); -const trace = @import("../../tracy.zig").trace; -const codegen = @import("../../codegen.zig"); - -const ErrorMsg = Zcu.ErrorMsg; -const Target = std.Target; - -const log = std.log.scoped(.riscv_codegen); -const tracking_log = std.log.scoped(.tracking); -const verbose_tracking_log = std.log.scoped(.verbose_tracking); -const wip_mir_log = std.log.scoped(.wip_mir); -const Alignment = InternPool.Alignment; - -const CodeGenError = codegen.CodeGenError; - -const bits = @import("bits.zig"); -const abi = @import("abi.zig"); -const Lower = @import("Lower.zig"); -const mnem_import = @import("mnem.zig"); -const Mnemonic = mnem_import.Mnemonic; -const Pseudo = mnem_import.Pseudo; -const encoding = @import("encoding.zig"); - -const Register = bits.Register; -const CSR = bits.CSR; -const Immediate = bits.Immediate; -const Memory = bits.Memory; -const FrameIndex = bits.FrameIndex; -const RegisterManager = abi.RegisterManager; -const RegisterLock = RegisterManager.RegisterLock; -const Instruction = encoding.Instruction; - -const InnerError = CodeGenError || error{OutOfRegisters}; - -pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { - return comptime &.initMany(&.{ - .expand_intcast_safe, - .expand_int_from_float_safe, - .expand_int_from_float_optimized_safe, - .expand_add_safe, - .expand_sub_safe, - .expand_mul_safe, - }); -} - -pt: Zcu.PerThread, -air: Air, -liveness: Air.Liveness, -bin_file: *link.File, -gpa: Allocator, - -mod: *Package.Module, -target: *const std.Target, -args: []MCValue, -ret_mcv: InstTracking, -func_index: InternPool.Index, -fn_type: Type, -arg_index: usize, -src_loc: Zcu.LazySrcLoc, - -mir_instructions: std.MultiArrayList(Mir.Inst) = .{}, - -owner: Owner, - -/// Byte offset within the source file of the ending curly. -end_di_line: u32, -end_di_column: u32, - -scope_generation: u32, - -/// The value is an offset into the `Function` `code` from the beginning. -/// To perform the reloc, write 32-bit signed little-endian integer -/// which is a relative jump, based on the address following the reloc. -exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, - -reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, - -/// Whenever there is a runtime branch, we push a Branch onto this stack, -/// and pop it off when the runtime branch joins. This provides an "overlay" -/// of the table of mappings from instructions to `MCValue` from within the branch. -/// This way we can modify the `MCValue` for an instruction in different ways -/// within different branches. Special consideration is needed when a branch -/// joins with its parent, to make sure all instructions have the same MCValue -/// across each runtime branch upon joining. -branch_stack: *std.array_list.Managed(Branch), - -// Currently set vector properties, null means they haven't been set yet in the function. -avl: ?u64, -vtype: ?bits.VType, - -// Key is the block instruction -blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, -register_manager: RegisterManager = .{}, - -const_tracking: ConstTrackingMap = .{}, -inst_tracking: InstTrackingMap = .{}, - -frame_allocs: std.MultiArrayList(FrameAlloc) = .{}, -free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty, -frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{}, - -loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { - /// The state to restore before branching. - state: State, - /// The branch target. - jmp_target: Mir.Inst.Index, -}) = .{}, - -/// Debug field, used to find bugs in the compiler. -air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init, - -const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; - -const SymbolOffset = struct { sym: u32, off: i32 = 0 }; -const RegisterOffset = struct { reg: Register, off: i32 = 0 }; -pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 }; - -const Owner = union(enum) { - nav_index: InternPool.Nav.Index, - lazy_sym: link.File.LazySymbol, - - fn getSymbolIndex(owner: Owner, func: *Func) !u32 { - const pt = func.pt; - switch (owner) { - .nav_index => |nav_index| { - const elf_file = func.bin_file.cast(.elf).?; - return elf_file.zigObjectPtr().?.getOrCreateMetadataForNav(pt.zcu, nav_index); - }, - .lazy_sym => |lazy_sym| { - const elf_file = func.bin_file.cast(.elf).?; - return elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err| - func.fail("{s} creating lazy symbol", .{@errorName(err)}); - }, - } - } -}; - -const MCValue = union(enum) { - /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. - /// TODO Look into deleting this tag and using `dead` instead, since every use - /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. - none, - /// Control flow will not allow this value to be observed. - unreach, - /// No more references to this value remain. - /// The payload is the value of scope_generation at the point where the death occurred - dead: u32, - /// The value is undefined. Contains a symbol index to an undefined constant. Null means - /// set the undefined value via immediate instead of a load. - undef: ?u32, - /// A pointer-sized integer that fits in a register. - /// If the type is a pointer, this is the pointer address in virtual address space. - immediate: u64, - /// The value doesn't exist in memory yet. - load_symbol: SymbolOffset, - /// The address of the memory location not-yet-allocated by the linker. - lea_symbol: SymbolOffset, - /// The value is in a target-specific register. - register: Register, - /// The value is split across two registers - register_pair: [2]Register, - /// The value is in memory at a hard-coded address. - /// If the type is a pointer, it means the pointer address is at this memory location. - memory: u64, - /// The value stored at an offset from a frame index - /// Payload is a frame address. - load_frame: FrameAddr, - /// The address of an offset from a frame index - /// Payload is a frame address. - lea_frame: FrameAddr, - air_ref: Air.Inst.Ref, - /// The value is in memory at a constant offset from the address in a register. - indirect: RegisterOffset, - /// The value is a constant offset from the value in a register. - register_offset: RegisterOffset, - /// This indicates that we have already allocated a frame index for this instruction, - /// but it has not been spilled there yet in the current control flow. - /// Payload is a frame index. - reserved_frame: FrameIndex, - - fn isMemory(mcv: MCValue) bool { - return switch (mcv) { - .memory, .indirect, .load_frame => true, - else => false, - }; - } - - fn isImmediate(mcv: MCValue) bool { - return switch (mcv) { - .immediate => true, - else => false, - }; - } - - fn isRegister(mcv: MCValue) bool { - return switch (mcv) { - .register => true, - .register_offset => |reg_off| return reg_off.off == 0, - else => false, - }; - } - - fn isMutable(mcv: MCValue) bool { - return switch (mcv) { - .none => unreachable, - .unreach => unreachable, - .dead => unreachable, - - .immediate, - .memory, - .lea_frame, - .undef, - .lea_symbol, - .air_ref, - .reserved_frame, - => false, - - .register, - .register_pair, - .register_offset, - .load_symbol, - .indirect, - => true, - - .load_frame => |frame_addr| !frame_addr.index.isNamed(), - }; - } - - fn address(mcv: MCValue) MCValue { - return switch (mcv) { - .none, - .unreach, - .dead, - .immediate, - .lea_frame, - .register_offset, - .register_pair, - .register, - .undef, - .air_ref, - .lea_symbol, - .reserved_frame, - => unreachable, // not in memory - - .load_symbol => |sym_off| .{ .lea_symbol = sym_off }, - .memory => |addr| .{ .immediate = addr }, - .load_frame => |off| .{ .lea_frame = off }, - .indirect => |reg_off| switch (reg_off.off) { - 0 => .{ .register = reg_off.reg }, - else => .{ .register_offset = reg_off }, - }, - }; - } - - fn deref(mcv: MCValue) MCValue { - return switch (mcv) { - .none, - .unreach, - .dead, - .memory, - .indirect, - .undef, - .air_ref, - .register_pair, - .load_frame, - .load_symbol, - .reserved_frame, - => unreachable, // not a pointer - - .immediate => |addr| .{ .memory = addr }, - .register => |reg| .{ .indirect = .{ .reg = reg } }, - .register_offset => |reg_off| .{ .indirect = reg_off }, - .lea_frame => |off| .{ .load_frame = off }, - .lea_symbol => |sym_off| .{ .load_symbol = sym_off }, - }; - } - - fn offset(mcv: MCValue, off: i32) MCValue { - return switch (mcv) { - .none, - .unreach, - .dead, - .undef, - .air_ref, - .reserved_frame, - => unreachable, // not valid - .register_pair, - .memory, - .indirect, - .load_symbol, - .lea_symbol, - => switch (off) { - 0 => mcv, - else => unreachable, - }, - .load_frame => |frame| .{ .load_frame = .{ .index = frame.index, .off = frame.off + off } }, - .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) }, - .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } }, - .register_offset => |reg_off| .{ .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off } }, - .lea_frame => |frame_addr| .{ - .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, - }, - }; - } - - fn getReg(mcv: MCValue) ?Register { - return switch (mcv) { - .register => |reg| reg, - .register_offset, .indirect => |ro| ro.reg, - else => null, - }; - } - - fn getRegs(mcv: *const MCValue) []const Register { - return switch (mcv.*) { - .register => |*reg| @as(*const [1]Register, reg), - .register_pair => |*regs| regs, - .register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg), - else => &.{}, - }; - } -}; - -const Branch = struct { - inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty, - - fn deinit(func: *Branch, gpa: Allocator) void { - func.inst_table.deinit(gpa); - func.* = undefined; - } -}; - -const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking); -const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking); - -const InstTracking = struct { - long: MCValue, - short: MCValue, - - fn init(result: MCValue) InstTracking { - return .{ .long = switch (result) { - .none, - .unreach, - .undef, - .immediate, - .memory, - .load_frame, - .lea_frame, - .load_symbol, - .lea_symbol, - => result, - .dead, - .reserved_frame, - .air_ref, - => unreachable, - .register, - .register_pair, - .register_offset, - .indirect, - => .none, - }, .short = result }; - } - - fn getReg(inst_tracking: InstTracking) ?Register { - return inst_tracking.short.getReg(); - } - - fn getRegs(inst_tracking: *const InstTracking) []const Register { - return inst_tracking.short.getRegs(); - } - - fn spill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void { - if (std.meta.eql(inst_tracking.long, inst_tracking.short)) return; // Already spilled - // Allocate or reuse frame index - switch (inst_tracking.long) { - .none => inst_tracking.long = try function.allocRegOrMem( - function.typeOfIndex(inst), - inst, - false, - ), - .load_frame => {}, - .reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } }, - else => unreachable, - } - tracking_log.debug("spill %{d} from {} to {}", .{ inst, inst_tracking.short, inst_tracking.long }); - try function.genCopy(function.typeOfIndex(inst), inst_tracking.long, inst_tracking.short); - } - - fn reuseFrame(inst_tracking: *InstTracking) void { - switch (inst_tracking.long) { - .reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } }, - else => {}, - } - inst_tracking.short = switch (inst_tracking.long) { - .none, - .unreach, - .undef, - .immediate, - .memory, - .load_frame, - .lea_frame, - .load_symbol, - .lea_symbol, - => inst_tracking.long, - .dead, - .register, - .register_pair, - .register_offset, - .indirect, - .reserved_frame, - .air_ref, - => unreachable, - }; - } - - fn trackSpill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void { - try function.freeValue(inst_tracking.short); - inst_tracking.reuseFrame(); - tracking_log.debug("%{d} => {f} (spilled)", .{ inst, inst_tracking.* }); - } - - fn verifyMaterialize(inst_tracking: InstTracking, target: InstTracking) void { - switch (inst_tracking.long) { - .none, - .unreach, - .undef, - .immediate, - .memory, - .lea_frame, - .load_symbol, - .lea_symbol, - => assert(std.meta.eql(inst_tracking.long, target.long)), - .load_frame, - .reserved_frame, - => switch (target.long) { - .none, - .load_frame, - .reserved_frame, - => {}, - else => unreachable, - }, - .dead, - .register, - .register_pair, - .register_offset, - .indirect, - .air_ref, - => unreachable, - } - } - - fn materialize( - inst_tracking: *InstTracking, - function: *Func, - inst: Air.Inst.Index, - target: InstTracking, - ) !void { - inst_tracking.verifyMaterialize(target); - try inst_tracking.materializeUnsafe(function, inst, target); - } - - fn materializeUnsafe( - inst_tracking: InstTracking, - function: *Func, - inst: Air.Inst.Index, - target: InstTracking, - ) !void { - const ty = function.typeOfIndex(inst); - if ((inst_tracking.long == .none or inst_tracking.long == .reserved_frame) and target.long == .load_frame) - try function.genCopy(ty, target.long, inst_tracking.short); - try function.genCopy(ty, target.short, inst_tracking.short); - } - - fn trackMaterialize(inst_tracking: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void { - inst_tracking.verifyMaterialize(target); - // Don't clobber reserved frame indices - inst_tracking.long = if (target.long == .none) switch (inst_tracking.long) { - .load_frame => |addr| .{ .reserved_frame = addr.index }, - .reserved_frame => inst_tracking.long, - else => target.long, - } else target.long; - inst_tracking.short = target.short; - tracking_log.debug("%{d} => {f} (materialize)", .{ inst, inst_tracking.* }); - } - - fn resurrect(inst_tracking: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void { - switch (inst_tracking.short) { - .dead => |die_generation| if (die_generation >= scope_generation) { - inst_tracking.reuseFrame(); - tracking_log.debug("%{d} => {f} (resurrect)", .{ inst, inst_tracking.* }); - }, - else => {}, - } - } - - fn die(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void { - if (inst_tracking.short == .dead) return; - try function.freeValue(inst_tracking.short); - inst_tracking.short = .{ .dead = function.scope_generation }; - tracking_log.debug("%{d} => {f} (death)", .{ inst, inst_tracking.* }); - } - - fn reuse( - inst_tracking: *InstTracking, - function: *Func, - new_inst: ?Air.Inst.Index, - old_inst: Air.Inst.Index, - ) void { - inst_tracking.short = .{ .dead = function.scope_generation }; - if (new_inst) |inst| - tracking_log.debug("%{d} => {f} (reuse %{d})", .{ inst, inst_tracking.*, old_inst }) - else - tracking_log.debug("tmp => {f} (reuse %{d})", .{ inst_tracking.*, old_inst }); - } - - fn liveOut(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) void { - for (inst_tracking.getRegs()) |reg| { - if (function.register_manager.isRegFree(reg)) { - tracking_log.debug("%{d} => {f} (live-out)", .{ inst, inst_tracking.* }); - continue; - } - - const index = RegisterManager.indexOfRegIntoTracked(reg).?; - const tracked_inst = function.register_manager.registers[index]; - const tracking = function.getResolvedInstValue(tracked_inst); - - // Disable death. - var found_reg = false; - var remaining_reg: Register = .zero; - for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) { - assert(!found_reg); - found_reg = true; - } else { - assert(remaining_reg == .zero); - remaining_reg = tracked_reg; - }; - assert(found_reg); - tracking.short = switch (remaining_reg) { - .zero => .{ .dead = function.scope_generation }, - else => .{ .register = remaining_reg }, - }; - - // Perform side-effects of freeValue manually. - function.register_manager.freeReg(reg); - - tracking_log.debug("%{d} => {f} (live-out %{d})", .{ inst, inst_tracking.*, tracked_inst }); - } - } - - pub fn format(inst_tracking: InstTracking, writer: *std.Io.Writer) std.Io.Writer.Error!void { - if (!std.meta.eql(inst_tracking.long, inst_tracking.short)) try writer.print("|{}| ", .{inst_tracking.long}); - try writer.print("{}", .{inst_tracking.short}); - } -}; - -const FrameAlloc = struct { - abi_size: u31, - spill_pad: u3, - abi_align: Alignment, - ref_count: u16, - - fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc { - return .{ - .abi_size = @intCast(alloc_abi.size), - .spill_pad = alloc_abi.pad, - .abi_align = alloc_abi.alignment, - .ref_count = 0, - }; - } - fn initType(ty: Type, zcu: *Zcu) FrameAlloc { - return init(.{ - .size = ty.abiSize(zcu), - .alignment = ty.abiAlignment(zcu), - }); - } - fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc { - const abi_size = ty.abiSize(zcu); - const spill_size = if (abi_size < 8) - math.ceilPowerOfTwoAssert(u64, abi_size) - else - std.mem.alignForward(u64, abi_size, 8); - return init(.{ - .size = spill_size, - .pad = @intCast(spill_size - abi_size), - .alignment = ty.abiAlignment(zcu).maxStrict( - Alignment.fromNonzeroByteUnits(@min(spill_size, 8)), - ), - }); - } -}; - -const BlockData = struct { - relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, - state: State, - - fn deinit(bd: *BlockData, gpa: Allocator) void { - bd.relocs.deinit(gpa); - bd.* = undefined; - } -}; - -const State = struct { - registers: RegisterManager.TrackedRegisters, - reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, - free_registers: RegisterManager.RegisterBitSet, - inst_tracking_len: u32, - scope_generation: u32, -}; - -fn initRetroactiveState(func: *Func) State { - var state: State = undefined; - state.inst_tracking_len = @intCast(func.inst_tracking.count()); - state.scope_generation = func.scope_generation; - return state; -} - -fn saveRetroactiveState(func: *Func, state: *State) !void { - const free_registers = func.register_manager.free_registers; - var it = free_registers.iterator(.{ .kind = .unset }); - while (it.next()) |index| { - const tracked_inst = func.register_manager.registers[index]; - state.registers[index] = tracked_inst; - state.reg_tracking[index] = func.inst_tracking.get(tracked_inst).?; - } - state.free_registers = free_registers; -} - -fn saveState(func: *Func) !State { - var state = func.initRetroactiveState(); - try func.saveRetroactiveState(&state); - return state; -} - -fn restoreState(func: *Func, state: State, deaths: []const Air.Inst.Index, comptime opts: struct { - emit_instructions: bool, - update_tracking: bool, - resurrect: bool, - close_scope: bool, -}) !void { - if (opts.close_scope) { - for ( - func.inst_tracking.keys()[state.inst_tracking_len..], - func.inst_tracking.values()[state.inst_tracking_len..], - ) |inst, *tracking| try tracking.die(func, inst); - func.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len); - } - - if (opts.resurrect) for ( - func.inst_tracking.keys()[0..state.inst_tracking_len], - func.inst_tracking.values()[0..state.inst_tracking_len], - ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation); - for (deaths) |death| try func.processDeath(death); - - const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - if (opts.update_tracking) {} else std.heap.stackFallback(@sizeOf(ExpectedContents), func.gpa); - - var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity( - stack.get(), - @typeInfo(ExpectedContents).array.len, - ); - defer if (!opts.update_tracking) { - for (reg_locks.items) |lock| func.register_manager.unlockReg(lock); - reg_locks.deinit(); - }; - - for (0..state.registers.len) |index| { - const current_maybe_inst = if (func.register_manager.free_registers.isSet(index)) - null - else - func.register_manager.registers[index]; - const target_maybe_inst = if (state.free_registers.isSet(index)) - null - else - state.registers[index]; - if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst| - assert(func.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len); - if (opts.emit_instructions) { - if (current_maybe_inst) |current_inst| { - try func.inst_tracking.getPtr(current_inst).?.spill(func, current_inst); - } - if (target_maybe_inst) |target_inst| { - const target_tracking = func.inst_tracking.getPtr(target_inst).?; - try target_tracking.materialize(func, target_inst, state.reg_tracking[index]); - } - } - if (opts.update_tracking) { - if (current_maybe_inst) |current_inst| { - try func.inst_tracking.getPtr(current_inst).?.trackSpill(func, current_inst); - } - blk: { - const inst = target_maybe_inst orelse break :blk; - const reg = RegisterManager.regAtTrackedIndex(@intCast(index)); - func.register_manager.freeReg(reg); - func.register_manager.getRegAssumeFree(reg, inst); - } - if (target_maybe_inst) |target_inst| { - func.inst_tracking.getPtr(target_inst).?.trackMaterialize( - target_inst, - state.reg_tracking[index], - ); - } - } else if (target_maybe_inst) |_| - try reg_locks.append(func.register_manager.lockRegIndexAssumeUnused(@intCast(index))); - } - - if (opts.update_tracking and std.debug.runtime_safety) { - assert(func.register_manager.free_registers.eql(state.free_registers)); - var used_reg_it = state.free_registers.iterator(.{ .kind = .unset }); - while (used_reg_it.next()) |index| - assert(func.register_manager.registers[index] == state.registers[index]); - } -} - -const Func = @This(); - -const CallView = enum(u1) { - callee, - caller, -}; - -pub fn generate( - bin_file: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - air: *const Air, - liveness: *const ?Air.Liveness, -) CodeGenError!Mir { - const zcu = pt.zcu; - const gpa = zcu.gpa; - const ip = &zcu.intern_pool; - const func = zcu.funcInfo(func_index); - const fn_type = Type.fromInterned(func.ty); - const mod = zcu.navFileScope(func.owner_nav).mod.?; - - var branch_stack = std.array_list.Managed(Branch).init(gpa); - defer { - assert(branch_stack.items.len == 1); - branch_stack.items[0].deinit(gpa); - branch_stack.deinit(); - } - try branch_stack.append(.{}); - - var function: Func = .{ - .gpa = gpa, - .air = air.*, - .pt = pt, - .mod = mod, - .bin_file = bin_file, - .liveness = liveness.*.?, - .target = &mod.resolved_target.result, - .owner = .{ .nav_index = func.owner_nav }, - .args = undefined, // populated after `resolveCallingConventionValues` - .ret_mcv = undefined, // populated after `resolveCallingConventionValues` - .func_index = func_index, - .fn_type = fn_type, - .arg_index = 0, - .branch_stack = &branch_stack, - .src_loc = src_loc, - .end_di_line = func.rbrace_line, - .end_di_column = func.rbrace_column, - .scope_generation = 0, - .avl = null, - .vtype = null, - }; - defer { - function.frame_allocs.deinit(gpa); - function.free_frame_indices.deinit(gpa); - function.frame_locs.deinit(gpa); - function.loops.deinit(gpa); - var block_it = function.blocks.valueIterator(); - while (block_it.next()) |block| block.deinit(gpa); - function.blocks.deinit(gpa); - function.inst_tracking.deinit(gpa); - function.const_tracking.deinit(gpa); - function.exitlude_jump_relocs.deinit(gpa); - function.mir_instructions.deinit(gpa); - } - - wip_mir_log.debug("{f}:", .{fmtNav(func.owner_nav, ip)}); - - try function.frame_allocs.resize(gpa, FrameIndex.named_count); - function.frame_allocs.set( - @intFromEnum(FrameIndex.stack_frame), - FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }), - ); - function.frame_allocs.set( - @intFromEnum(FrameIndex.call_frame), - FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }), - ); - - const fn_info = zcu.typeToFunc(fn_type).?; - var call_info = function.resolveCallingConventionValues(fn_info, &.{}) catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - else => |e| return e, - }; - - defer call_info.deinit(&function); - - function.args = call_info.args; - function.ret_mcv = call_info.return_value; - function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{ - .size = Type.u64.abiSize(zcu), - .alignment = Type.u64.abiAlignment(zcu).min(call_info.stack_align), - })); - function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{ - .size = Type.u64.abiSize(zcu), - .alignment = Alignment.min( - call_info.stack_align, - Alignment.fromNonzeroByteUnits(function.target.stackAlignment()), - ), - })); - function.frame_allocs.set(@intFromEnum(FrameIndex.args_frame), FrameAlloc.init(.{ - .size = call_info.stack_byte_count, - .alignment = call_info.stack_align, - })); - function.frame_allocs.set(@intFromEnum(FrameIndex.spill_frame), FrameAlloc.init(.{ - .size = 0, - .alignment = Type.u64.abiAlignment(zcu), - })); - - function.gen() catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), - else => |e| return e, - }; - - var mir: Mir = .{ - .instructions = function.mir_instructions.toOwnedSlice(), - .frame_locs = function.frame_locs.toOwnedSlice(), - }; - errdefer mir.deinit(gpa); - return mir; -} - -pub fn generateLazy( - bin_file: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - lazy_sym: link.File.LazySymbol, - atom_index: u32, - w: *std.Io.Writer, - debug_output: link.File.DebugInfoOutput, -) (CodeGenError || std.Io.Writer.Error)!void { - _ = atom_index; - const comp = bin_file.comp; - const gpa = comp.gpa; - const mod = comp.root_mod; - - var function: Func = .{ - .gpa = gpa, - .air = undefined, - .pt = pt, - .mod = mod, - .bin_file = bin_file, - .liveness = undefined, - .target = &mod.resolved_target.result, - .owner = .{ .lazy_sym = lazy_sym }, - .args = undefined, // populated after `resolveCallingConventionValues` - .ret_mcv = undefined, // populated after `resolveCallingConventionValues` - .func_index = undefined, - .fn_type = undefined, - .arg_index = 0, - .branch_stack = undefined, - .src_loc = src_loc, - .end_di_line = undefined, - .end_di_column = undefined, - .scope_generation = 0, - .avl = null, - .vtype = null, - }; - defer function.mir_instructions.deinit(gpa); - - function.genLazy(lazy_sym) catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), - else => |e| return e, - }; - - var mir: Mir = .{ - .instructions = function.mir_instructions.toOwnedSlice(), - .frame_locs = function.frame_locs.toOwnedSlice(), - }; - defer mir.deinit(gpa); - - var emit: Emit = .{ - .lower = .{ - .pt = pt, - .allocator = gpa, - .mir = mir, - .cc = .auto, - .src_loc = src_loc, - .output_mode = comp.config.output_mode, - .link_mode = comp.config.link_mode, - .pic = mod.pic, - }, - .bin_file = bin_file, - .debug_output = debug_output, - .w = w, - .prev_di_pc = undefined, // no debug info yet - .prev_di_line = undefined, // no debug info yet - .prev_di_column = undefined, // no debug info yet - }; - defer emit.deinit(); - - emit.emitMir() catch |err| switch (err) { - error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?), - error.InvalidInstruction => |e| return function.fail("emit MIR failed: {s} (Zig compiler bug)", .{@errorName(e)}), - else => |e| return e, - }; -} - -const FormatWipMirData = struct { - func: *Func, - inst: Mir.Inst.Index, -}; -fn formatWipMir(data: FormatWipMirData, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const pt = data.func.pt; - const comp = pt.zcu.comp; - var lower: Lower = .{ - .pt = pt, - .allocator = data.func.gpa, - .mir = .{ - .instructions = data.func.mir_instructions.slice(), - .frame_locs = data.func.frame_locs.slice(), - }, - .cc = .auto, - .src_loc = data.func.src_loc, - .output_mode = comp.config.output_mode, - .link_mode = comp.config.link_mode, - .pic = comp.root_mod.pic, - }; - var first = true; - for ((lower.lowerMir(data.inst, .{ .allow_frame_locs = false }) catch |err| switch (err) { - error.LowerFail => { - defer { - lower.err_msg.?.deinit(data.func.gpa); - lower.err_msg = null; - } - try writer.writeAll(lower.err_msg.?.msg); - return; - }, - error.OutOfMemory, error.InvalidInstruction => |e| { - try writer.writeAll(switch (e) { - error.OutOfMemory => "Out of memory", - error.InvalidInstruction => "CodeGen failed to find a viable instruction.", - }); - return; - }, - else => |e| return e, - }).insts) |lowered_inst| { - if (!first) try writer.writeAll("\ndebug(wip_mir): "); - try writer.print(" | {}", .{lowered_inst}); - first = false; - } -} -fn fmtWipMir(func: *Func, inst: Mir.Inst.Index) std.fmt.Alt(FormatWipMirData, formatWipMir) { - return .{ .data = .{ .func = func, .inst = inst } }; -} - -const FormatNavData = struct { - ip: *const InternPool, - nav_index: InternPool.Nav.Index, -}; -fn formatNav(data: FormatNavData, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("{f}", .{data.ip.getNav(data.nav_index).fqn.fmt(data.ip)}); -} -fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Alt(FormatNavData, formatNav) { - return .{ .data = .{ - .ip = ip, - .nav_index = nav_index, - } }; -} - -const FormatAirData = struct { - func: *Func, - inst: Air.Inst.Index, -}; -fn formatAir(data: FormatAirData, writer: *std.Io.Writer) std.Io.Writer.Error!void { - // Not acceptable implementation because it ignores `writer`: - //data.func.air.dumpInst(data.inst, data.func.pt, data.func.liveness); - _ = data; - _ = writer; - @panic("unimplemented"); -} -fn fmtAir(func: *Func, inst: Air.Inst.Index) std.fmt.Alt(FormatAirData, formatAir) { - return .{ .data = .{ .func = func, .inst = inst } }; -} - -const FormatTrackingData = struct { - func: *Func, -}; -fn formatTracking(data: FormatTrackingData, writer: *std.Io.Writer) std.Io.Writer.Error!void { - var it = data.func.inst_tracking.iterator(); - while (it.next()) |entry| try writer.print("\n%{d} = {f}", .{ entry.key_ptr.*, entry.value_ptr.* }); -} -fn fmtTracking(func: *Func) std.fmt.Alt(FormatTrackingData, formatTracking) { - return .{ .data = .{ .func = func } }; -} - -fn addInst(func: *Func, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { - const gpa = func.gpa; - try func.mir_instructions.ensureUnusedCapacity(gpa, 1); - const result_index: Mir.Inst.Index = @intCast(func.mir_instructions.len); - func.mir_instructions.appendAssumeCapacity(inst); - if (switch (inst.tag) { - else => true, - .pseudo_dbg_prologue_end, - .pseudo_dbg_line_column, - .pseudo_dbg_epilogue_begin, - .pseudo_dead, - => false, - }) wip_mir_log.debug("{f}", .{func.fmtWipMir(result_index)}); - return result_index; -} - -fn addPseudo(func: *Func, mnem: Mnemonic) error{OutOfMemory}!Mir.Inst.Index { - return func.addInst(.{ - .tag = mnem, - .data = .none, - }); -} - -/// Returns a temporary register that contains the value of the `reg` csr. -/// -/// Caller's duty to lock the return register is needed. -fn getCsr(func: *Func, csr: CSR) !Register { - assert(func.hasFeature(.zicsr)); - const dst_reg = try func.register_manager.allocReg(null, func.regTempClassForType(Type.u64)); - _ = try func.addInst(.{ - .tag = .csrrs, - .data = .{ .csr = .{ - .csr = csr, - .rd = dst_reg, - .rs1 = .x0, - } }, - }); - return dst_reg; -} - -fn setVl(func: *Func, dst_reg: Register, avl: u64, options: bits.VType) !void { - if (func.avl == avl) if (func.vtype) |vtype| { - // it's already set, we don't need to do anything - if (@as(u8, @bitCast(vtype)) == @as(u8, @bitCast(options))) return; - }; - - func.avl = avl; - func.vtype = options; - - if (avl == 0) { - // the caller means to do "vsetvli zero, zero ..." which keeps the avl to whatever it was before - const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); - _ = try func.addInst(.{ - .tag = .vsetvli, - .data = .{ .i_type = .{ - .rd = dst_reg, - .rs1 = .zero, - .imm12 = Immediate.u(options_int), - } }, - }); - } else { - // if the avl can fit into u5 we can use vsetivli otherwise use vsetvli - if (avl <= std.math.maxInt(u5)) { - const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options)); - _ = try func.addInst(.{ - .tag = .vsetivli, - .data = .{ - .i_type = .{ - .rd = dst_reg, - .rs1 = @enumFromInt(avl), - .imm12 = Immediate.u(options_int), - }, - }, - }); - } else { - const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); - const temp_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = avl }); - _ = try func.addInst(.{ - .tag = .vsetvli, - .data = .{ .i_type = .{ - .rd = dst_reg, - .rs1 = temp_reg, - .imm12 = Immediate.u(options_int), - } }, - }); - } - } -} - -const required_features = [_]Target.riscv.Feature{ - .d, - .m, - .a, - .zicsr, - .v, - .zbb, -}; - -fn gen(func: *Func) !void { - const pt = func.pt; - const zcu = pt.zcu; - const fn_info = zcu.typeToFunc(func.fn_type).?; - - inline for (required_features) |feature| { - if (!func.hasFeature(feature)) { - return func.fail( - "target missing required feature {s}", - .{@tagName(feature)}, - ); - } - } - - if (fn_info.cc != .naked) { - _ = try func.addPseudo(.pseudo_dbg_prologue_end); - - const backpatch_stack_alloc = try func.addPseudo(.pseudo_dead); - const backpatch_ra_spill = try func.addPseudo(.pseudo_dead); - const backpatch_fp_spill = try func.addPseudo(.pseudo_dead); - const backpatch_fp_add = try func.addPseudo(.pseudo_dead); - const backpatch_spill_callee_preserved_regs = try func.addPseudo(.pseudo_dead); - - switch (func.ret_mcv.long) { - .none, .unreach => {}, - .indirect => { - // The address where to store the return value for the caller is in a - // register which the callee is free to clobber. Therefore, we purposely - // spill it to stack immediately. - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(Type.u64, zcu)); - try func.genSetMem( - .{ .frame = frame_index }, - 0, - Type.u64, - func.ret_mcv.long.address().offset(-func.ret_mcv.short.indirect.off), - ); - func.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } }; - tracking_log.debug("spill {} to {}", .{ func.ret_mcv.long, frame_index }); - }, - else => unreachable, - } - - try func.genBody(func.air.getMainBody()); - - for (func.exitlude_jump_relocs.items) |jmp_reloc| { - func.mir_instructions.items(.data)[jmp_reloc].j_type.inst = - @intCast(func.mir_instructions.len); - } - - _ = try func.addPseudo(.pseudo_dbg_epilogue_begin); - - const backpatch_restore_callee_preserved_regs = try func.addPseudo(.pseudo_dead); - const backpatch_ra_restore = try func.addPseudo(.pseudo_dead); - const backpatch_fp_restore = try func.addPseudo(.pseudo_dead); - const backpatch_stack_alloc_restore = try func.addPseudo(.pseudo_dead); - - // ret - _ = try func.addInst(.{ - .tag = .jalr, - .data = .{ .i_type = .{ - .rd = .zero, - .rs1 = .ra, - .imm12 = Immediate.s(0), - } }, - }); - - const frame_layout = try func.computeFrameLayout(); - const need_save_reg = frame_layout.save_reg_list.count() > 0; - - func.mir_instructions.set(backpatch_stack_alloc, .{ - .tag = .addi, - .data = .{ .i_type = .{ - .rd = .sp, - .rs1 = .sp, - .imm12 = Immediate.s(-@as(i32, @intCast(frame_layout.stack_adjust))), - } }, - }); - func.mir_instructions.set(backpatch_ra_spill, .{ - .tag = .pseudo_store_rm, - .data = .{ .rm = .{ - .r = .ra, - .m = .{ - .base = .{ .frame = .ret_addr }, - .mod = .{ .size = .dword, .unsigned = false }, - }, - } }, - }); - func.mir_instructions.set(backpatch_ra_restore, .{ - .tag = .pseudo_load_rm, - .data = .{ .rm = .{ - .r = .ra, - .m = .{ - .base = .{ .frame = .ret_addr }, - .mod = .{ .size = .dword, .unsigned = false }, - }, - } }, - }); - func.mir_instructions.set(backpatch_fp_spill, .{ - .tag = .pseudo_store_rm, - .data = .{ .rm = .{ - .r = .s0, - .m = .{ - .base = .{ .frame = .base_ptr }, - .mod = .{ .size = .dword, .unsigned = false }, - }, - } }, - }); - func.mir_instructions.set(backpatch_fp_restore, .{ - .tag = .pseudo_load_rm, - .data = .{ .rm = .{ - .r = .s0, - .m = .{ - .base = .{ .frame = .base_ptr }, - .mod = .{ .size = .dword, .unsigned = false }, - }, - } }, - }); - func.mir_instructions.set(backpatch_fp_add, .{ - .tag = .addi, - .data = .{ .i_type = .{ - .rd = .s0, - .rs1 = .sp, - .imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)), - } }, - }); - func.mir_instructions.set(backpatch_stack_alloc_restore, .{ - .tag = .addi, - .data = .{ .i_type = .{ - .rd = .sp, - .rs1 = .sp, - .imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)), - } }, - }); - - if (need_save_reg) { - func.mir_instructions.set(backpatch_spill_callee_preserved_regs, .{ - .tag = .pseudo_spill_regs, - .data = .{ .reg_list = frame_layout.save_reg_list }, - }); - - func.mir_instructions.set(backpatch_restore_callee_preserved_regs, .{ - .tag = .pseudo_restore_regs, - .data = .{ .reg_list = frame_layout.save_reg_list }, - }); - } - } else { - _ = try func.addPseudo(.pseudo_dbg_prologue_end); - try func.genBody(func.air.getMainBody()); - _ = try func.addPseudo(.pseudo_dbg_epilogue_begin); - } - - // Drop them off at the rbrace. - _ = try func.addInst(.{ - .tag = .pseudo_dbg_line_column, - .data = .{ .pseudo_dbg_line_column = .{ - .line = func.end_di_line, - .column = func.end_di_column, - } }, - }); -} - -fn genLazy(func: *Func, lazy_sym: link.File.LazySymbol) InnerError!void { - const pt = func.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) { - .@"enum" => { - const enum_ty = Type.fromInterned(lazy_sym.ty); - wip_mir_log.debug("{f}.@tagName:", .{enum_ty.fmt(pt)}); - - const param_regs = abi.Registers.Integer.function_arg_regs; - const ret_reg = param_regs[0]; - const enum_mcv: MCValue = .{ .register = param_regs[1] }; - - const exitlude_jump_relocs = try func.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu)); - defer func.gpa.free(exitlude_jump_relocs); - - const data_reg, const data_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(data_lock); - - const elf_file = func.bin_file.cast(.elf).?; - const zo = elf_file.zigObjectPtr().?; - const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, .{ - .kind = .const_data, - .ty = enum_ty.toIntern(), - }) catch |err| - return func.fail("{s} creating lazy symbol", .{@errorName(err)}); - - try func.genSetReg(Type.u64, data_reg, .{ .lea_symbol = .{ .sym = sym_index } }); - - const cmp_reg, const cmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(cmp_lock); - - var data_off: i32 = 0; - const tag_names = enum_ty.enumFields(zcu); - for (exitlude_jump_relocs, 0..) |*exitlude_jump_reloc, tag_index| { - const tag_name_len = tag_names.get(ip)[tag_index].length(ip); - const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index)); - const tag_mcv = try func.genTypedValue(tag_val); - - _ = try func.genBinOp( - .cmp_neq, - enum_mcv, - enum_ty, - tag_mcv, - enum_ty, - cmp_reg, - ); - const skip_reloc = try func.condBr(Type.bool, .{ .register = cmp_reg }); - - try func.genSetMem( - .{ .reg = ret_reg }, - 0, - Type.u64, - .{ .register_offset = .{ .reg = data_reg, .off = data_off } }, - ); - - try func.genSetMem( - .{ .reg = ret_reg }, - 8, - Type.u64, - .{ .immediate = tag_name_len }, - ); - - exitlude_jump_reloc.* = try func.addInst(.{ - .tag = .pseudo_j, - .data = .{ .j_type = .{ - .rd = .zero, - .inst = undefined, - } }, - }); - func.performReloc(skip_reloc); - - data_off += @intCast(tag_name_len + 1); - } - - try func.airTrap(); - - for (exitlude_jump_relocs) |reloc| func.performReloc(reloc); - - _ = try func.addInst(.{ - .tag = .jalr, - .data = .{ .i_type = .{ - .rd = .zero, - .rs1 = .ra, - .imm12 = Immediate.s(0), - } }, - }); - }, - else => return func.fail( - "TODO implement {s} for {f}", - .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, - ), - } -} - -fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { - const pt = func.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const air_tags = func.air.instructions.items(.tag); - - for (body) |inst| { - if (func.liveness.isUnused(inst) and !func.air.mustLower(inst, ip)) continue; - wip_mir_log.debug("{f}", .{func.fmtAir(inst)}); - verbose_tracking_log.debug("{f}", .{func.fmtTracking()}); - - const old_air_bookkeeping = func.air_bookkeeping; - try func.ensureProcessDeathCapacity(Air.Liveness.bpi); - - func.reused_operands = @TypeOf(func.reused_operands).initEmpty(); - try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1); - const tag = air_tags[@intFromEnum(inst)]; - switch (tag) { - // zig fmt: off - .add, - .add_wrap, - .sub, - .sub_wrap, - - .add_sat, - - .mul, - .mul_wrap, - .div_trunc, - .div_exact, - .rem, - - .shl, .shl_exact, - .shr, .shr_exact, - - .bool_and, - .bool_or, - .bit_and, - .bit_or, - - .xor, - - .min, - .max, - => try func.airBinOp(inst, tag), - - - .ptr_add, - .ptr_sub => try func.airPtrArithmetic(inst, tag), - - .mod, - .div_float, - .div_floor, - => return func.fail("TODO: {s}", .{@tagName(tag)}), - - .sqrt, - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .floor, - .ceil, - .round, - .trunc_float, - .neg, - => try func.airUnaryMath(inst, tag), - - .add_with_overflow => try func.airAddWithOverflow(inst), - .sub_with_overflow => try func.airSubWithOverflow(inst), - .mul_with_overflow => try func.airMulWithOverflow(inst), - .shl_with_overflow => try func.airShlWithOverflow(inst), - - - .sub_sat => try func.airSubSat(inst), - .mul_sat => try func.airMulSat(inst), - .shl_sat => try func.airShlSat(inst), - - .add_safe, - .sub_safe, - .mul_safe, - .intcast_safe, - .int_from_float_safe, - .int_from_float_optimized_safe, - => return func.fail("TODO implement safety_checked_instructions", .{}), - - .cmp_lt, - .cmp_lte, - .cmp_eq, - .cmp_gte, - .cmp_gt, - .cmp_neq, - => try func.airCmp(inst, tag), - - .cmp_vector => try func.airCmpVector(inst), - .cmp_lt_errors_len => try func.airCmpLtErrorsLen(inst), - - .slice => try func.airSlice(inst), - .array_to_slice => try func.airArrayToSlice(inst), - - .slice_ptr => try func.airSlicePtr(inst), - .slice_len => try func.airSliceLen(inst), - - .alloc => try func.airAlloc(inst), - .ret_ptr => try func.airRetPtr(inst), - .arg => try func.airArg(inst), - .assembly => try func.airAsm(inst), - .bitcast => try func.airBitCast(inst), - .block => try func.airBlock(inst), - .br => try func.airBr(inst), - .repeat => try func.airRepeat(inst), - .switch_dispatch => try func.airSwitchDispatch(inst), - .trap => try func.airTrap(), - .breakpoint => try func.airBreakpoint(), - .ret_addr => try func.airRetAddr(inst), - .frame_addr => try func.airFrameAddress(inst), - .cond_br => try func.airCondBr(inst), - .dbg_stmt => try func.airDbgStmt(inst), - .dbg_empty_stmt => func.finishAirBookkeeping(), - .fptrunc => try func.airFptrunc(inst), - .fpext => try func.airFpext(inst), - .intcast => try func.airIntCast(inst), - .trunc => try func.airTrunc(inst), - .is_non_null => try func.airIsNonNull(inst), - .is_non_null_ptr => try func.airIsNonNullPtr(inst), - .is_null => try func.airIsNull(inst), - .is_null_ptr => try func.airIsNullPtr(inst), - .is_non_err => try func.airIsNonErr(inst), - .is_non_err_ptr => try func.airIsNonErrPtr(inst), - .is_err => try func.airIsErr(inst), - .is_err_ptr => try func.airIsErrPtr(inst), - .load => try func.airLoad(inst), - .loop => try func.airLoop(inst), - .not => try func.airNot(inst), - .ret => try func.airRet(inst, false), - .ret_safe => try func.airRet(inst, true), - .ret_load => try func.airRetLoad(inst), - .store => try func.airStore(inst, false), - .store_safe => try func.airStore(inst, true), - .struct_field_ptr=> try func.airStructFieldPtr(inst), - .struct_field_val=> try func.airStructFieldVal(inst), - .float_from_int => try func.airFloatFromInt(inst), - .int_from_float => try func.airIntFromFloat(inst), - .cmpxchg_strong => try func.airCmpxchg(inst, .strong), - .cmpxchg_weak => try func.airCmpxchg(inst, .weak), - .atomic_rmw => try func.airAtomicRmw(inst), - .atomic_load => try func.airAtomicLoad(inst), - .memcpy => try func.airMemcpy(inst), - .memmove => try func.airMemmove(inst), - .memset => try func.airMemset(inst, false), - .memset_safe => try func.airMemset(inst, true), - .set_union_tag => try func.airSetUnionTag(inst), - .get_union_tag => try func.airGetUnionTag(inst), - .clz => try func.airClz(inst), - .ctz => try func.airCtz(inst), - .popcount => try func.airPopcount(inst), - .abs => try func.airAbs(inst), - .byte_swap => try func.airByteSwap(inst), - .bit_reverse => try func.airBitReverse(inst), - .tag_name => try func.airTagName(inst), - .error_name => try func.airErrorName(inst), - .splat => try func.airSplat(inst), - .select => try func.airSelect(inst), - .shuffle_one => try func.airShuffleOne(inst), - .shuffle_two => try func.airShuffleTwo(inst), - .reduce => try func.airReduce(inst), - .aggregate_init => try func.airAggregateInit(inst), - .union_init => try func.airUnionInit(inst), - .prefetch => try func.airPrefetch(inst), - .mul_add => try func.airMulAdd(inst), - .addrspace_cast => return func.fail("TODO: addrspace_cast", .{}), - - .@"try" => try func.airTry(inst), - .try_cold => try func.airTry(inst), - .try_ptr => return func.fail("TODO: try_ptr", .{}), - .try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}), - - .dbg_var_ptr, - .dbg_var_val, - .dbg_arg_inline, - => try func.airDbgVar(inst), - - .dbg_inline_block => try func.airDbgInlineBlock(inst), - - .call => try func.airCall(inst, .auto), - .call_always_tail => try func.airCall(inst, .always_tail), - .call_never_tail => try func.airCall(inst, .never_tail), - .call_never_inline => try func.airCall(inst, .never_inline), - - .atomic_store_unordered => try func.airAtomicStore(inst, .unordered), - .atomic_store_monotonic => try func.airAtomicStore(inst, .monotonic), - .atomic_store_release => try func.airAtomicStore(inst, .release), - .atomic_store_seq_cst => try func.airAtomicStore(inst, .seq_cst), - .struct_field_ptr_index_0 => try func.airStructFieldPtrIndex(inst, 0), - .struct_field_ptr_index_1 => try func.airStructFieldPtrIndex(inst, 1), - .struct_field_ptr_index_2 => try func.airStructFieldPtrIndex(inst, 2), - .struct_field_ptr_index_3 => try func.airStructFieldPtrIndex(inst, 3), - - .field_parent_ptr => try func.airFieldParentPtr(inst), - - .switch_br => try func.airSwitchBr(inst), - .loop_switch_br => try func.airLoopSwitchBr(inst), - - .ptr_slice_len_ptr => try func.airPtrSliceLenPtr(inst), - .ptr_slice_ptr_ptr => try func.airPtrSlicePtrPtr(inst), - - .array_elem_val => try func.airArrayElemVal(inst), - - .slice_elem_val => try func.airSliceElemVal(inst), - .slice_elem_ptr => try func.airSliceElemPtr(inst), - - .ptr_elem_val => try func.airPtrElemVal(inst), - .ptr_elem_ptr => try func.airPtrElemPtr(inst), - - .inferred_alloc, .inferred_alloc_comptime => unreachable, - .unreach => func.finishAirBookkeeping(), - - .optional_payload => try func.airOptionalPayload(inst), - .optional_payload_ptr => try func.airOptionalPayloadPtr(inst), - .optional_payload_ptr_set => try func.airOptionalPayloadPtrSet(inst), - .unwrap_errunion_err => try func.airUnwrapErrErr(inst), - .unwrap_errunion_payload => try func.airUnwrapErrPayload(inst), - .unwrap_errunion_err_ptr => try func.airUnwrapErrErrPtr(inst), - .unwrap_errunion_payload_ptr=> try func.airUnwrapErrPayloadPtr(inst), - .errunion_payload_ptr_set => try func.airErrUnionPayloadPtrSet(inst), - .err_return_trace => try func.airErrReturnTrace(inst), - .set_err_return_trace => try func.airSetErrReturnTrace(inst), - .save_err_return_trace_index=> try func.airSaveErrReturnTraceIndex(inst), - - .wrap_optional => try func.airWrapOptional(inst), - .wrap_errunion_payload => try func.airWrapErrUnionPayload(inst), - .wrap_errunion_err => try func.airWrapErrUnionErr(inst), - - .runtime_nav_ptr => try func.airRuntimeNavPtr(inst), - - .add_optimized, - .sub_optimized, - .mul_optimized, - .div_float_optimized, - .div_trunc_optimized, - .div_floor_optimized, - .div_exact_optimized, - .rem_optimized, - .mod_optimized, - .neg_optimized, - .cmp_lt_optimized, - .cmp_lte_optimized, - .cmp_eq_optimized, - .cmp_gte_optimized, - .cmp_gt_optimized, - .cmp_neq_optimized, - .cmp_vector_optimized, - .reduce_optimized, - .int_from_float_optimized, - => return func.fail("TODO implement optimized float mode", .{}), - - .is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}), - .error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}), - .vector_store_elem => return func.fail("TODO implement vector_store_elem", .{}), - - .c_va_arg => return func.fail("TODO implement c_va_arg", .{}), - .c_va_copy => return func.fail("TODO implement c_va_copy", .{}), - .c_va_end => return func.fail("TODO implement c_va_end", .{}), - .c_va_start => return func.fail("TODO implement c_va_start", .{}), - - .wasm_memory_size => unreachable, - .wasm_memory_grow => unreachable, - - .work_item_id => unreachable, - .work_group_size => unreachable, - .work_group_id => unreachable, - // zig fmt: on - } - - assert(!func.register_manager.lockedRegsExist()); - - if (std.debug.runtime_safety) { - if (func.air_bookkeeping < old_air_bookkeeping + 1) { - std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] }); - } - - { // check consistency of tracked registers - var it = func.register_manager.free_registers.iterator(.{ .kind = .unset }); - while (it.next()) |index| { - const tracked_inst = func.register_manager.registers[index]; - tracking_log.debug("tracked inst: {f}", .{tracked_inst}); - const tracking = func.getResolvedInstValue(tracked_inst); - for (tracking.getRegs()) |reg| { - if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; - } else return std.debug.panic( - \\%{f} takes up these regs: {any}, however this regs {any}, don't use it - , .{ tracked_inst, tracking.getRegs(), RegisterManager.regAtTrackedIndex(@intCast(index)) }); - } - } - } - } - verbose_tracking_log.debug("{f}", .{func.fmtTracking()}); -} - -fn getValue(func: *Func, value: MCValue, inst: ?Air.Inst.Index) !void { - for (value.getRegs()) |reg| try func.register_manager.getReg(reg, inst); -} - -fn getValueIfFree(func: *Func, value: MCValue, inst: ?Air.Inst.Index) void { - for (value.getRegs()) |reg| if (func.register_manager.isRegFree(reg)) - func.register_manager.getRegAssumeFree(reg, inst); -} - -fn freeValue(func: *Func, value: MCValue) !void { - switch (value) { - .register => |reg| func.register_manager.freeReg(reg), - .register_pair => |regs| for (regs) |reg| func.register_manager.freeReg(reg), - .register_offset => |reg_off| func.register_manager.freeReg(reg_off.reg), - else => {}, // TODO process stack allocation death - } -} - -fn feed(func: *Func, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void { - if (bt.feed()) if (operand.toIndex()) |inst| { - log.debug("feed inst: %{f}", .{inst}); - try func.processDeath(inst); - }; -} - -/// Asserts there is already capacity to insert into top branch inst_table. -fn processDeath(func: *Func, inst: Air.Inst.Index) !void { - try func.inst_tracking.getPtr(inst).?.die(func, inst); -} - -/// Called when there are no operands, and the instruction is always unreferenced. -fn finishAirBookkeeping(func: *Func) void { - if (std.debug.runtime_safety) { - func.air_bookkeeping += 1; - } -} - -fn finishAirResult(func: *Func, inst: Air.Inst.Index, result: MCValue) void { - if (func.liveness.isUnused(inst)) switch (result) { - .none, .dead, .unreach => {}, - // Why didn't the result die? - .register => |r| if (r != .zero) unreachable, - else => unreachable, - } else { - switch (result) { - .register => |r| if (r == .zero) unreachable, // Why did we discard a used result? - else => {}, - } - - tracking_log.debug("%{d} => {} (birth)", .{ inst, result }); - func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result)); - // In some cases, an operand may be reused as the result. - // If that operand died and was a register, it was freed by - // processDeath, so we have to "re-allocate" the register. - func.getValueIfFree(result, inst); - } - func.finishAirBookkeeping(); -} - -fn finishAir( - func: *Func, - inst: Air.Inst.Index, - result: MCValue, - operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, -) !void { - const tomb_bits = func.liveness.getTombBits(inst); - for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; - if (func.reused_operands.isSet(op_index)) continue; - try func.processDeath(op.toIndexAllowNone() orelse continue); - } - func.finishAirResult(inst, result); -} - -const FrameLayout = struct { - stack_adjust: i12, - save_reg_list: Mir.RegisterList, -}; - -fn setFrameLoc( - func: *Func, - frame_index: FrameIndex, - base: Register, - offset: *i32, - comptime aligned: bool, -) void { - const frame_i = @intFromEnum(frame_index); - if (aligned) { - const alignment: InternPool.Alignment = func.frame_allocs.items(.abi_align)[frame_i]; - offset.* = math.sign(offset.*) * @as(i32, @intCast(alignment.backward(@intCast(@abs(offset.*))))); - } - func.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* }); - offset.* += func.frame_allocs.items(.abi_size)[frame_i]; -} - -fn computeFrameLayout(func: *Func) !FrameLayout { - const frame_allocs_len = func.frame_allocs.len; - try func.frame_locs.resize(func.gpa, frame_allocs_len); - const stack_frame_order = try func.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count); - defer func.gpa.free(stack_frame_order); - - const frame_size = func.frame_allocs.items(.abi_size); - const frame_align = func.frame_allocs.items(.abi_align); - - for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index| - frame_order.* = @enumFromInt(frame_index); - - { - const SortContext = struct { - frame_align: @TypeOf(frame_align), - pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool { - return context.frame_align[@intFromEnum(lhs)].compare(.gt, context.frame_align[@intFromEnum(rhs)]); - } - }; - const sort_context = SortContext{ .frame_align = frame_align }; - mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan); - } - - var save_reg_list = Mir.RegisterList{}; - for (abi.Registers.all_preserved) |reg| { - if (func.register_manager.isRegAllocated(reg)) { - save_reg_list.push(&abi.Registers.all_preserved, reg); - } - } - - const total_alloc_size: i32 = blk: { - var i: i32 = 0; - for (stack_frame_order) |frame_index| { - i += frame_size[@intFromEnum(frame_index)]; - } - break :blk i; - }; - - const saved_reg_size = save_reg_list.size(); - frame_size[@intFromEnum(FrameIndex.spill_frame)] = @intCast(saved_reg_size); - - // The total frame size is calculated by the amount of s registers you need to save * 8, as each - // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0 - // register. Finally we align the frame size to the alignment of the base pointer. - const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)]; - const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)]; - const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)]; - - // TODO: this 64 should be a 16, but we were clobbering the top and bottom of the frame. - // maybe everything can go from the bottom? - const acc_frame_size: i32 = std.mem.alignForward( - i32, - total_alloc_size + 64 + args_frame_size + spill_frame_size + call_frame_size, - @intCast(frame_align[@intFromEnum(FrameIndex.base_ptr)].toByteUnits().?), - ); - log.debug("frame size: {d}", .{acc_frame_size}); - - // store the ra at total_size - 8, so it's the very first thing in the stack - // relative to the fp - func.frame_locs.set( - @intFromEnum(FrameIndex.ret_addr), - .{ .base = .sp, .disp = acc_frame_size - 8 }, - ); - func.frame_locs.set( - @intFromEnum(FrameIndex.base_ptr), - .{ .base = .sp, .disp = acc_frame_size - 16 }, - ); - - // now we grow the stack frame from the bottom of total frame in order to - // not need to know the size of the first allocation. Stack offsets point at the "bottom" - // of variables. - var s0_offset: i32 = -acc_frame_size; - func.setFrameLoc(.stack_frame, .s0, &s0_offset, true); - for (stack_frame_order) |frame_index| func.setFrameLoc(frame_index, .s0, &s0_offset, true); - func.setFrameLoc(.args_frame, .s0, &s0_offset, true); - func.setFrameLoc(.call_frame, .s0, &s0_offset, true); - func.setFrameLoc(.spill_frame, .s0, &s0_offset, true); - - return .{ - .stack_adjust = @intCast(acc_frame_size), - .save_reg_list = save_reg_list, - }; -} - -fn ensureProcessDeathCapacity(func: *Func, additional_count: usize) !void { - const table = &func.branch_stack.items[func.branch_stack.items.len - 1].inst_table; - try table.ensureUnusedCapacity(func.gpa, additional_count); -} - -fn memSize(func: *Func, ty: Type) Memory.Size { - const pt = func.pt; - const zcu = pt.zcu; - return switch (ty.zigTypeTag(zcu)) { - .float => Memory.Size.fromBitSize(ty.floatBits(func.target)), - else => Memory.Size.fromByteSize(ty.abiSize(zcu)), - }; -} - -fn splitType(func: *Func, ty: Type) ![2]Type { - const zcu = func.pt.zcu; - const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none); - var parts: [2]Type = undefined; - if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| { - part.* = switch (class) { - .integer => switch (part_i) { - 0 => Type.u64, - 1 => part: { - const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; - const elem_ty = try func.pt.intType(.unsigned, @intCast(elem_size * 8)); - break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) { - 1 => elem_ty, - else => |len| try func.pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }), - }; - }, - else => unreachable, - }, - else => return func.fail("TODO: splitType class {}", .{class}), - }; - } else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts; - return func.fail("TODO implement splitType for {f}", .{ty.fmt(func.pt)}); -} - -/// Truncates the value in the register in place. -/// Clobbers any remaining bits. -fn truncateRegister(func: *Func, ty: Type, reg: Register) !void { - const pt = func.pt; - const zcu = pt.zcu; - const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{ - .signedness = .unsigned, - .bits = @intCast(ty.bitSize(zcu)), - }; - assert(reg.class() == .int); - - const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return; - switch (int_info.signedness) { - .signed => { - _ = try func.addInst(.{ - .tag = .slli, - - .data = .{ - .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(shift), - }, - }, - }); - _ = try func.addInst(.{ - .tag = .srai, - - .data = .{ - .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(shift), - }, - }, - }); - }, - .unsigned => { - const mask = ~@as(u64, 0) >> shift; - if (mask < 256) { - _ = try func.addInst(.{ - .tag = .andi, - - .data = .{ - .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(@intCast(mask)), - }, - }, - }); - } else { - _ = try func.addInst(.{ - .tag = .slli, - - .data = .{ - .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(shift), - }, - }, - }); - _ = try func.addInst(.{ - .tag = .srli, - - .data = .{ - .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(shift), - }, - }, - }); - } - }, - } -} - -fn allocFrameIndex(func: *Func, alloc: FrameAlloc) !FrameIndex { - const frame_allocs_slice = func.frame_allocs.slice(); - const frame_size = frame_allocs_slice.items(.abi_size); - const frame_align = frame_allocs_slice.items(.abi_align); - - const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)]; - stack_frame_align.* = stack_frame_align.max(alloc.abi_align); - - for (func.free_frame_indices.keys(), 0..) |frame_index, free_i| { - const abi_size = frame_size[@intFromEnum(frame_index)]; - if (abi_size != alloc.abi_size) continue; - const abi_align = &frame_align[@intFromEnum(frame_index)]; - abi_align.* = abi_align.max(alloc.abi_align); - - _ = func.free_frame_indices.swapRemoveAt(free_i); - return frame_index; - } - const frame_index: FrameIndex = @enumFromInt(func.frame_allocs.len); - try func.frame_allocs.append(func.gpa, alloc); - log.debug("allocated frame {}", .{frame_index}); - return frame_index; -} - -/// Use a pointer instruction as the basis for allocating stack memory. -fn allocMemPtr(func: *Func, inst: Air.Inst.Index) !FrameIndex { - const pt = func.pt; - const zcu = pt.zcu; - const ptr_ty = func.typeOfIndex(inst); - const val_ty = ptr_ty.childType(zcu); - return func.allocFrameIndex(FrameAlloc.init(.{ - .size = math.cast(u32, val_ty.abiSize(zcu)) orelse { - return func.fail("type '{f}' too big to fit into stack frame", .{val_ty.fmt(pt)}); - }, - .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"), - })); -} - -fn typeRegClass(func: *Func, ty: Type) abi.RegisterClass { - const pt = func.pt; - const zcu = pt.zcu; - return switch (ty.zigTypeTag(zcu)) { - .float => .float, - .vector => .vector, - else => .int, - }; -} - -fn regGeneralClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet { - return switch (ty.zigTypeTag(func.pt.zcu)) { - .float => abi.Registers.Float.general_purpose, - .vector => abi.Registers.Vector.general_purpose, - else => abi.Registers.Integer.general_purpose, - }; -} - -fn regTempClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet { - return switch (ty.zigTypeTag(func.pt.zcu)) { - .float => abi.Registers.Float.temporary, - .vector => abi.Registers.Vector.general_purpose, // there are no temporary vector registers - else => abi.Registers.Integer.temporary, - }; -} - -fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue { - const pt = func.pt; - const zcu = pt.zcu; - - const bit_size = elem_ty.bitSize(zcu); - const min_size: u64 = switch (elem_ty.zigTypeTag(zcu)) { - .float => if (func.hasFeature(.d)) 64 else 32, - .vector => 256, // TODO: calculate it from avl * vsew - else => 64, - }; - - if (reg_ok and bit_size <= min_size) { - if (func.register_manager.tryAllocReg(inst, func.regGeneralClassForType(elem_ty))) |reg| { - return .{ .register = reg }; - } - } else if (reg_ok and elem_ty.zigTypeTag(zcu) == .vector) { - return func.fail("did you forget to extend vector registers before allocating", .{}); - } - - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(elem_ty, zcu)); - return .{ .load_frame = .{ .index = frame_index } }; -} - -/// Allocates a register from the general purpose set and returns the Register and the Lock. -/// -/// Up to the caller to unlock the register later. -fn allocReg(func: *Func, reg_class: abi.RegisterClass) !struct { Register, RegisterLock } { - if (reg_class == .float and !func.hasFeature(.f)) - std.debug.panic("allocReg class == float where F isn't enabled", .{}); - if (reg_class == .vector and !func.hasFeature(.v)) - std.debug.panic("allocReg class == vector where V isn't enabled", .{}); - - const class = switch (reg_class) { - .int => abi.Registers.Integer.general_purpose, - .float => abi.Registers.Float.general_purpose, - .vector => abi.Registers.Vector.general_purpose, - }; - - const reg = try func.register_manager.allocReg(null, class); - const lock = func.register_manager.lockRegAssumeUnused(reg); - return .{ reg, lock }; -} - -/// Similar to `allocReg` but will copy the MCValue into the Register unless `operand` is already -/// a register, in which case it will return a possible lock to that register. -fn promoteReg(func: *Func, ty: Type, operand: MCValue) !struct { Register, ?RegisterLock } { - if (operand == .register) { - const op_reg = operand.register; - return .{ op_reg, func.register_manager.lockReg(operand.register) }; - } - - const class = func.typeRegClass(ty); - const reg, const lock = try func.allocReg(class); - try func.genSetReg(ty, reg, operand); - return .{ reg, lock }; -} - -fn elemOffset(func: *Func, index_ty: Type, index: MCValue, elem_size: u64) !Register { - const reg: Register = blk: { - switch (index) { - .immediate => |imm| { - // Optimisation: if index MCValue is an immediate, we can multiply in `comptime` - // and set the register directly to the scaled offset as an immediate. - const reg = try func.register_manager.allocReg(null, func.regGeneralClassForType(index_ty)); - try func.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size }); - break :blk reg; - }, - else => { - const reg = try func.copyToTmpRegister(index_ty, index); - const lock = func.register_manager.lockRegAssumeUnused(reg); - defer func.register_manager.unlockReg(lock); - - const result_reg, const result_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(result_lock); - - try func.genBinOp( - .mul, - .{ .register = reg }, - index_ty, - .{ .immediate = elem_size }, - index_ty, - result_reg, - ); - - break :blk result_reg; - }, - } - }; - return reg; -} - -pub fn spillInstruction(func: *Func, reg: Register, inst: Air.Inst.Index) !void { - const tracking = func.inst_tracking.getPtr(inst) orelse return; - for (tracking.getRegs()) |tracked_reg| { - if (tracked_reg.id() == reg.id()) break; - } else unreachable; // spilled reg not tracked with spilled instruciton - try tracking.spill(func, inst); - try tracking.trackSpill(func, inst); -} - -pub fn spillRegisters(func: *Func, comptime registers: []const Register) !void { - inline for (registers) |reg| try func.register_manager.getKnownReg(reg, null); -} - -/// Copies a value to a register without tracking the register. The register is not considered -/// allocated. A second call to `copyToTmpRegister` may return the same register. -/// This can have a side effect of spilling instructions to the stack to free up a register. -fn copyToTmpRegister(func: *Func, ty: Type, mcv: MCValue) !Register { - log.debug("copyToTmpRegister ty: {f}", .{ty.fmt(func.pt)}); - const reg = try func.register_manager.allocReg(null, func.regTempClassForType(ty)); - try func.genSetReg(ty, reg, mcv); - return reg; -} - -/// Allocates a new register and copies `mcv` into it. -/// `reg_owner` is the instruction that gets associated with the register in the register table. -/// This can have a side effect of spilling instructions to the stack to free up a register. -fn copyToNewRegister(func: *Func, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const ty = func.typeOfIndex(reg_owner); - const reg = try func.register_manager.allocReg(reg_owner, func.regGeneralClassForType(ty)); - try func.genSetReg(func.typeOfIndex(reg_owner), reg, mcv); - return MCValue{ .register = reg }; -} - -fn airAlloc(func: *Func, inst: Air.Inst.Index) !void { - const result = MCValue{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } }; - return func.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airRetPtr(func: *Func, inst: Air.Inst.Index) !void { - const result: MCValue = switch (func.ret_mcv.long) { - .none => .{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } }, - .load_frame => .{ .register_offset = .{ - .reg = (try func.copyToNewRegister( - inst, - func.ret_mcv.long, - )).register, - .off = func.ret_mcv.short.indirect.off, - } }, - else => |t| return func.fail("TODO: airRetPtr {s}", .{@tagName(t)}), - }; - return func.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airFptrunc(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFptrunc for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airFpext(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFpext for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIntCast(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const src_ty = func.typeOf(ty_op.operand); - const dst_ty = func.typeOfIndex(inst); - - const result: MCValue = result: { - const src_int_info = src_ty.intInfo(zcu); - const dst_int_info = dst_ty.intInfo(zcu); - - const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; - - const src_mcv = try func.resolveInst(ty_op.operand); - - const src_storage_bits: u16 = switch (src_mcv) { - .register => 64, - .load_frame => src_int_info.bits, - else => return func.fail("airIntCast from {s}", .{@tagName(src_mcv)}), - }; - - const dst_mcv = if (dst_int_info.bits <= src_storage_bits and - math.divCeil(u16, dst_int_info.bits, 64) catch unreachable == - math.divCeil(u32, src_storage_bits, 64) catch unreachable and - func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { - const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true); - try func.genCopy(min_ty, dst_mcv, src_mcv); - break :dst dst_mcv; - }; - - if (dst_int_info.bits <= src_int_info.bits) - break :result dst_mcv; - - if (dst_int_info.bits > 64 or src_int_info.bits > 64) - break :result null; // TODO - - break :result dst_mcv; - } orelse return func.fail("TODO: implement airIntCast from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airTrunc(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - if (func.liveness.isUnused(inst)) - return func.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none }); - // we assume no zeroext in the "Zig ABI", so it's fine to just not truncate it. - const operand = try func.resolveInst(ty_op.operand); - - // we can do it just to be safe, but this shouldn't be needed for no-runtime safety modes - switch (operand) { - .register => |reg| try func.truncateRegister(func.typeOf(ty_op.operand), reg), - else => {}, - } - - return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none }); -} - -fn airNot(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const pt = func.pt; - const zcu = pt.zcu; - - const operand = try func.resolveInst(ty_op.operand); - const ty = func.typeOf(ty_op.operand); - - const operand_reg, const operand_lock = try func.promoteReg(ty, operand); - defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); - - const dst_reg: Register = - if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register) - operand.register - else - (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register; - - switch (ty.zigTypeTag(zcu)) { - .bool => { - _ = try func.addInst(.{ - .tag = .pseudo_not, - .data = .{ - .rr = .{ - .rs = operand_reg, - .rd = dst_reg, - }, - }, - }); - }, - .int => { - const size = ty.bitSize(zcu); - if (!math.isPowerOfTwo(size)) - return func.fail("TODO: airNot non-pow 2 int size", .{}); - - switch (size) { - 32, 64 => { - _ = try func.addInst(.{ - .tag = .xori, - .data = .{ - .i_type = .{ - .rd = dst_reg, - .rs1 = operand_reg, - .imm12 = Immediate.s(-1), - }, - }, - }); - }, - 8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}), - else => unreachable, - } - }, - else => unreachable, - } - - break :result .{ .register = dst_reg }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSlice(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data; - - const slice_ty = func.typeOfIndex(inst); - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu)); - - const ptr_ty = func.typeOf(bin_op.lhs); - try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }); - - const len_ty = func.typeOf(bin_op.rhs); - try func.genSetMem( - .{ .frame = frame_index }, - @intCast(ptr_ty.abiSize(zcu)), - len_ty, - .{ .air_ref = bin_op.rhs }, - ); - - const result = MCValue{ .load_frame = .{ .index = frame_index } }; - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airBinOp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const pt = func.pt; - const zcu = pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); - - const dst_ty = func.typeOfIndex(inst); - if (dst_ty.isAbiInt(zcu)) { - const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const bit_size: u32 = @intCast(dst_ty.bitSize(zcu)); - if (abi_size * 8 > bit_size) { - const dst_lock = switch (dst_mcv) { - .register => |dst_reg| func.register_manager.lockRegAssumeUnused(dst_reg), - else => null, - }; - defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); - - if (dst_mcv.isRegister()) { - try func.truncateRegister(dst_ty, dst_mcv.getReg().?); - } else { - const tmp_reg, const tmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(tmp_lock); - - const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1)); - const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref(); - try func.genSetReg(hi_ty, tmp_reg, hi_mcv); - try func.truncateRegister(dst_ty, tmp_reg); - try func.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg }); - } - } - } - - return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn binOp( - func: *Func, - maybe_inst: ?Air.Inst.Index, - air_tag: Air.Inst.Tag, - lhs_air: Air.Inst.Ref, - rhs_air: Air.Inst.Ref, -) !MCValue { - _ = maybe_inst; - const pt = func.pt; - const zcu = pt.zcu; - const lhs_ty = func.typeOf(lhs_air); - const rhs_ty = func.typeOf(rhs_air); - - if (lhs_ty.isRuntimeFloat()) libcall: { - const float_bits = lhs_ty.floatBits(func.target); - const type_needs_libcall = switch (float_bits) { - 16 => true, - 32, 64 => false, - 80, 128 => true, - else => unreachable, - }; - if (!type_needs_libcall) break :libcall; - return func.fail("binOp libcall runtime-float ops", .{}); - } - - // don't have support for certain sizes of addition - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => {}, // works differently and fails in a different place - else => if (lhs_ty.bitSize(zcu) > 64) return func.fail("TODO: binOp >= 64 bits", .{}), - } - - const lhs_mcv = try func.resolveInst(lhs_air); - const rhs_mcv = try func.resolveInst(rhs_air); - - const class_for_dst_ty: abi.RegisterClass = switch (air_tag) { - // will always return int register no matter the input - .cmp_eq, - .cmp_neq, - .cmp_lt, - .cmp_lte, - .cmp_gt, - .cmp_gte, - => .int, - - else => func.typeRegClass(lhs_ty), - }; - - const dst_reg, const dst_lock = try func.allocReg(class_for_dst_ty); - defer func.register_manager.unlockReg(dst_lock); - - try func.genBinOp( - air_tag, - lhs_mcv, - lhs_ty, - rhs_mcv, - rhs_ty, - dst_reg, - ); - - return .{ .register = dst_reg }; -} - -/// Does the same thing as binOp however is meant to be used internally to the backend. -/// -/// The `dst_reg` argument is meant to be caller-locked. Asserts that the binOp result can be -/// fit into the register. -/// -/// Assumes that the `dst_reg` class is correct. -fn genBinOp( - func: *Func, - tag: Air.Inst.Tag, - lhs_mcv: MCValue, - lhs_ty: Type, - rhs_mcv: MCValue, - rhs_ty: Type, - dst_reg: Register, -) !void { - const pt = func.pt; - const zcu = pt.zcu; - const bit_size = lhs_ty.bitSize(zcu); - - const is_unsigned = lhs_ty.isUnsignedInt(zcu); - - const lhs_reg, const maybe_lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv); - const rhs_reg, const maybe_rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv); - - defer if (maybe_lhs_lock) |lock| func.register_manager.unlockReg(lock); - defer if (maybe_rhs_lock) |lock| func.register_manager.unlockReg(lock); - - switch (tag) { - .add, - .add_wrap, - .sub, - .sub_wrap, - .mul, - .mul_wrap, - .rem, - .div_trunc, - .div_exact, - => { - switch (tag) { - .rem, - .div_trunc, - .div_exact, - => { - if (!math.isPowerOfTwo(bit_size)) { - try func.truncateRegister(lhs_ty, lhs_reg); - try func.truncateRegister(rhs_ty, rhs_reg); - } - }, - else => { - if (!math.isPowerOfTwo(bit_size)) - return func.fail( - "TODO: genBinOp verify if needs to truncate {s} non-pow 2, found {}", - .{ @tagName(tag), bit_size }, - ); - }, - } - - switch (lhs_ty.zigTypeTag(zcu)) { - .int => { - const mnem: Mnemonic = switch (tag) { - .add, .add_wrap => switch (bit_size) { - 8, 16, 64 => .add, - 32 => .addw, - else => unreachable, - }, - .sub, .sub_wrap => switch (bit_size) { - 8, 16, 32 => .subw, - 64 => .sub, - else => unreachable, - }, - .mul, .mul_wrap => switch (bit_size) { - 8, 16, 64 => .mul, - 32 => .mulw, - else => unreachable, - }, - .rem => switch (bit_size) { - 8, 16, 32 => if (is_unsigned) .remuw else .remw, - else => if (is_unsigned) .remu else .rem, - }, - .div_trunc, .div_exact => switch (bit_size) { - 8, 16, 32 => if (is_unsigned) .divuw else .divw, - else => if (is_unsigned) .divu else .div, - }, - else => unreachable, - }; - - _ = try func.addInst(.{ - .tag = mnem, - .data = .{ - .r_type = .{ - .rd = dst_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - }, - }, - }); - }, - .float => { - const mir_tag: Mnemonic = switch (tag) { - .add => switch (bit_size) { - 32 => .fadds, - 64 => .faddd, - else => unreachable, - }, - .sub => switch (bit_size) { - 32 => .fsubs, - 64 => .fsubd, - else => unreachable, - }, - .mul => switch (bit_size) { - 32 => .fmuls, - 64 => .fmuld, - else => unreachable, - }, - else => return func.fail("TODO: genBinOp {s} Float", .{@tagName(tag)}), - }; - - _ = try func.addInst(.{ - .tag = mir_tag, - .data = .{ - .r_type = .{ - .rd = dst_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - }, - }, - }); - }, - .vector => { - const num_elem = lhs_ty.vectorLen(zcu); - const elem_size = lhs_ty.childType(zcu).bitSize(zcu); - - const child_ty = lhs_ty.childType(zcu); - - const mir_tag: Mnemonic = switch (tag) { - .add => switch (child_ty.zigTypeTag(zcu)) { - .int => .vaddvv, - .float => .vfaddvv, - else => unreachable, - }, - .sub => switch (child_ty.zigTypeTag(zcu)) { - .int => .vsubvv, - .float => .vfsubvv, - else => unreachable, - }, - .mul => switch (child_ty.zigTypeTag(zcu)) { - .int => .vmulvv, - .float => .vfmulvv, - else => unreachable, - }, - else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}), - }; - - try func.setVl(.zero, num_elem, .{ - .vsew = switch (elem_size) { - 8 => .@"8", - 16 => .@"16", - 32 => .@"32", - 64 => .@"64", - else => return func.fail("TODO: genBinOp > 64 bit elements, found {d}", .{elem_size}), - }, - .vlmul = .m1, - .vma = true, - .vta = true, - }); - - _ = try func.addInst(.{ - .tag = mir_tag, - .data = .{ - .r_type = .{ - .rd = dst_reg, - .rs1 = rhs_reg, - .rs2 = lhs_reg, - }, - }, - }); - }, - else => unreachable, - } - }, - - .add_sat, - => { - if (bit_size != 64 or !is_unsigned) - return func.fail("TODO: genBinOp ty: {f}", .{lhs_ty.fmt(pt)}); - - const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg }); - const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg); - defer func.register_manager.unlockReg(tmp_lock); - - _ = try func.addInst(.{ - .tag = .add, - .data = .{ .r_type = .{ - .rd = tmp_reg, - .rs1 = rhs_reg, - .rs2 = lhs_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .sltu, - .data = .{ .r_type = .{ - .rd = dst_reg, - .rs1 = tmp_reg, - .rs2 = lhs_reg, - } }, - }); - - // neg dst_reg, dst_reg - _ = try func.addInst(.{ - .tag = .sub, - .data = .{ .r_type = .{ - .rd = dst_reg, - .rs1 = .zero, - .rs2 = dst_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .@"or", - .data = .{ .r_type = .{ - .rd = dst_reg, - .rs1 = dst_reg, - .rs2 = tmp_reg, - } }, - }); - }, - - .ptr_add, - .ptr_sub, - => { - const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg }); - const tmp_mcv = MCValue{ .register = tmp_reg }; - const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg); - defer func.register_manager.unlockReg(tmp_lock); - - // RISC-V has no immediate mul, so we copy the size to a temporary register - const elem_size = lhs_ty.elemType2(zcu).abiSize(zcu); - const elem_size_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = elem_size }); - - try func.genBinOp( - .mul, - tmp_mcv, - rhs_ty, - .{ .register = elem_size_reg }, - Type.u64, - tmp_reg, - ); - - try func.genBinOp( - switch (tag) { - .ptr_add => .add, - .ptr_sub => .sub, - else => unreachable, - }, - lhs_mcv, - Type.u64, // we know it's a pointer, so it'll be usize. - tmp_mcv, - Type.u64, - dst_reg, - ); - }, - - .bit_and, - .bit_or, - .bool_and, - .bool_or, - => { - _ = try func.addInst(.{ - .tag = switch (tag) { - .bit_and, .bool_and => .@"and", - .bit_or, .bool_or => .@"or", - else => unreachable, - }, - .data = .{ - .r_type = .{ - .rd = dst_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - }, - }, - }); - - switch (tag) { - .bool_and, - .bool_or, - => try func.truncateRegister(Type.bool, dst_reg), - else => {}, - } - }, - - .shr, - .shr_exact, - .shl, - .shl_exact, - => { - if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) return func.fail("TODO: vector shift with scalar rhs", .{}); - if (bit_size > 64) return func.fail("TODO: genBinOp shift > 64 bits, {}", .{bit_size}); - try func.truncateRegister(rhs_ty, rhs_reg); - - const mir_tag: Mnemonic = switch (tag) { - .shl, .shl_exact => switch (bit_size) { - 1...31, 33...64 => .sll, - 32 => .sllw, - else => unreachable, - }, - .shr, .shr_exact => switch (bit_size) { - 1...31, 33...64 => .srl, - 32 => .srlw, - else => unreachable, - }, - else => unreachable, - }; - - _ = try func.addInst(.{ - .tag = mir_tag, - .data = .{ .r_type = .{ - .rd = dst_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - } }, - }); - }, - - // TODO: move the isel logic out of lower and into here. - .cmp_eq, - .cmp_neq, - .cmp_lt, - .cmp_lte, - .cmp_gt, - .cmp_gte, - => { - assert(lhs_reg.class() == rhs_reg.class()); - if (lhs_reg.class() == .int) { - try func.truncateRegister(lhs_ty, lhs_reg); - try func.truncateRegister(rhs_ty, rhs_reg); - } - - _ = try func.addInst(.{ - .tag = .pseudo_compare, - .data = .{ - .compare = .{ - .op = switch (tag) { - .cmp_eq => .eq, - .cmp_neq => .neq, - .cmp_lt => .lt, - .cmp_lte => .lte, - .cmp_gt => .gt, - .cmp_gte => .gte, - else => unreachable, - }, - .rd = dst_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - .ty = lhs_ty, - }, - }, - }); - }, - - // A branchless @min/@max sequence. - // - // Assume that a0 and a1 are the lhs and rhs respectively. - // Also assume that a2 is the destination register. - // - // Algorithm: - // slt s0, a0, a1 - // sub s0, zero, s0 - // xor a2, a0, a1 - // and s0, a2, s0 - // xor a2, a0, s0 # a0 is @min, a1 is @max - // - // "slt s0, a0, a1" will set s0 to 1 if a0 is less than a1, and 1 otherwise. - // - // "sub s0, zero, s0" will set all the bits of s0 to 1 if it was 1, otherwise it'll remain at 0. - // - // "xor a2, a0, a1" stores the bitwise XOR of a0 and a1 in a2. Effectively getting the difference between them. - // - // "and a0, a2, s0" here we mask the result of the XOR with the negated s0. If a0 < a1, s0 is -1, which - // doesn't change the bits of a2. If a0 >= a1, s0 is 0, nullifying a2. - // - // "xor a2, a0, s0" the final XOR operation adjusts a2 to be the minimum value of a0 and a1. If a0 was less than - // a1, s0 was -1, flipping all the bits in a2 and effectively restoring a0. If a0 was greater than or equal to a1, - // s0 was 0, leaving a2 unchanged as a0. - .min, .max => { - switch (lhs_ty.zigTypeTag(zcu)) { - .int => { - const int_info = lhs_ty.intInfo(zcu); - - const mask_reg, const mask_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(mask_lock); - - _ = try func.addInst(.{ - .tag = if (int_info.signedness == .unsigned) .sltu else .slt, - .data = .{ .r_type = .{ - .rd = mask_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .sub, - .data = .{ .r_type = .{ - .rd = mask_reg, - .rs1 = .zero, - .rs2 = mask_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .xor, - .data = .{ .r_type = .{ - .rd = dst_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .@"and", - .data = .{ .r_type = .{ - .rd = mask_reg, - .rs1 = dst_reg, - .rs2 = mask_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .xor, - .data = .{ .r_type = .{ - .rd = dst_reg, - .rs1 = if (tag == .min) rhs_reg else lhs_reg, - .rs2 = mask_reg, - } }, - }); - }, - else => |t| return func.fail("TODO: genBinOp min/max for {s}", .{@tagName(t)}), - } - }, - else => return func.fail("TODO: genBinOp {}", .{tag}), - } -} - -fn airPtrArithmetic(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data; - const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); - return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airAddWithOverflow(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; - - const rhs_ty = func.typeOf(extra.rhs); - const lhs_ty = func.typeOf(extra.lhs); - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => return func.fail("TODO implement add with overflow for Vector type", .{}), - .int => { - const int_info = lhs_ty.intInfo(zcu); - - const tuple_ty = func.typeOfIndex(inst); - const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); - const offset = result_mcv.load_frame; - - if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { - const add_result = try func.binOp(null, .add, extra.lhs, extra.rhs); - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), - lhs_ty, - add_result, - ); - - const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result); - const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg); - defer func.register_manager.unlockReg(trunc_reg_lock); - - const overflow_reg, const overflow_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(overflow_lock); - - // if the result isn't equal after truncating it to the given type, - // an overflow must have happened. - try func.truncateRegister(lhs_ty, trunc_reg); - try func.genBinOp( - .cmp_neq, - add_result, - lhs_ty, - .{ .register = trunc_reg }, - rhs_ty, - overflow_reg, - ); - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), - Type.u1, - .{ .register = overflow_reg }, - ); - - break :result result_mcv; - } else { - const rhs_mcv = try func.resolveInst(extra.rhs); - const lhs_mcv = try func.resolveInst(extra.lhs); - - const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv); - const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv); - defer { - if (rhs_lock) |lock| func.register_manager.unlockReg(lock); - if (lhs_lock) |lock| func.register_manager.unlockReg(lock); - } - - try func.truncateRegister(rhs_ty, rhs_reg); - try func.truncateRegister(lhs_ty, lhs_reg); - - const dest_reg, const dest_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(dest_lock); - - _ = try func.addInst(.{ - .tag = .add, - .data = .{ .r_type = .{ - .rs1 = rhs_reg, - .rs2 = lhs_reg, - .rd = dest_reg, - } }, - }); - - try func.truncateRegister(func.typeOfIndex(inst), dest_reg); - const add_result: MCValue = .{ .register = dest_reg }; - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), - lhs_ty, - add_result, - ); - - const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result); - const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg); - defer func.register_manager.unlockReg(trunc_reg_lock); - - const overflow_reg, const overflow_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(overflow_lock); - - // if the result isn't equal after truncating it to the given type, - // an overflow must have happened. - try func.truncateRegister(lhs_ty, trunc_reg); - try func.genBinOp( - .cmp_neq, - add_result, - lhs_ty, - .{ .register = trunc_reg }, - rhs_ty, - overflow_reg, - ); - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), - Type.u1, - .{ .register = overflow_reg }, - ); - - break :result result_mcv; - } - }, - else => unreachable, - } - }; - - return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airSubWithOverflow(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const lhs = try func.resolveInst(extra.lhs); - const rhs = try func.resolveInst(extra.rhs); - const lhs_ty = func.typeOf(extra.lhs); - const rhs_ty = func.typeOf(extra.rhs); - - const int_info = lhs_ty.intInfo(zcu); - - if (!math.isPowerOfTwo(int_info.bits) or int_info.bits < 8) { - return func.fail("TODO: airSubWithOverflow non-power of 2 and less than 8 bits", .{}); - } - - if (int_info.bits > 64) { - return func.fail("TODO: airSubWithOverflow > 64 bits", .{}); - } - - const tuple_ty = func.typeOfIndex(inst); - const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); - const offset = result_mcv.load_frame; - - const dest_mcv = try func.binOp(null, .sub, extra.lhs, extra.rhs); - assert(dest_mcv == .register); - const dest_reg = dest_mcv.register; - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), - lhs_ty, - .{ .register = dest_reg }, - ); - - const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs); - defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock); - - const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs); - defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock); - - const overflow_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = 0 }); - - const overflow_lock = func.register_manager.lockRegAssumeUnused(overflow_reg); - defer func.register_manager.unlockReg(overflow_lock); - - switch (int_info.signedness) { - .unsigned => { - _ = try func.addInst(.{ - .tag = .sltu, - .data = .{ .r_type = .{ - .rd = overflow_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - } }, - }); - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), - Type.u1, - .{ .register = overflow_reg }, - ); - - break :result result_mcv; - }, - .signed => { - switch (int_info.bits) { - 64 => { - _ = try func.addInst(.{ - .tag = .slt, - .data = .{ .r_type = .{ - .rd = overflow_reg, - .rs1 = overflow_reg, - .rs2 = rhs_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .slt, - .data = .{ .r_type = .{ - .rd = rhs_reg, - .rs1 = rhs_reg, - .rs2 = lhs_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .xor, - .data = .{ .r_type = .{ - .rd = lhs_reg, - .rs1 = overflow_reg, - .rs2 = rhs_reg, - } }, - }); - - try func.genBinOp( - .cmp_neq, - .{ .register = overflow_reg }, - Type.u64, - .{ .register = rhs_reg }, - Type.u64, - overflow_reg, - ); - - try func.genSetMem( - .{ .frame = offset.index }, - offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), - Type.u1, - .{ .register = overflow_reg }, - ); - - break :result result_mcv; - }, - else => |int_bits| return func.fail("TODO: airSubWithOverflow signed {}", .{int_bits}), - } - }, - } - }; - - return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airMulWithOverflow(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const lhs = try func.resolveInst(extra.lhs); - const rhs = try func.resolveInst(extra.rhs); - const lhs_ty = func.typeOf(extra.lhs); - const rhs_ty = func.typeOf(extra.rhs); - - const tuple_ty = func.typeOfIndex(inst); - - // genSetReg needs to support register_offset src_mcv for this to be true. - const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); - - const result_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu)); - const overflow_off: i32 = @intCast(tuple_ty.structFieldOffset(1, zcu)); - - const dest_reg, const dest_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(dest_lock); - - try func.genBinOp( - .mul, - lhs, - lhs_ty, - rhs, - rhs_ty, - dest_reg, - ); - - try func.genCopy( - lhs_ty, - result_mcv.offset(result_off), - .{ .register = dest_reg }, - ); - - switch (lhs_ty.zigTypeTag(zcu)) { - else => |x| return func.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}), - .int => { - if (std.debug.runtime_safety) assert(lhs_ty.eql(rhs_ty, zcu)); - - const trunc_reg = try func.copyToTmpRegister(lhs_ty, .{ .register = dest_reg }); - const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg); - defer func.register_manager.unlockReg(trunc_reg_lock); - - const overflow_reg, const overflow_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(overflow_lock); - - // if the result isn't equal after truncating it to the given type, - // an overflow must have happened. - try func.truncateRegister(func.typeOf(extra.lhs), trunc_reg); - try func.genBinOp( - .cmp_neq, - .{ .register = dest_reg }, - lhs_ty, - .{ .register = trunc_reg }, - rhs_ty, - overflow_reg, - ); - - try func.genCopy( - lhs_ty, - result_mcv.offset(overflow_off), - .{ .register = overflow_reg }, - ); - - break :result result_mcv; - }, - } - }; - - return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airShlWithOverflow(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (func.liveness.isUnused(inst)) - .unreach - else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu)) - return func.fail("TODO implement vector airShlWithOverflow with scalar rhs", .{}) - else - return func.fail("TODO implement airShlWithOverflow", .{}); - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSubSat(func: *Func, inst: Air.Inst.Index) !void { - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSubSat", .{}); - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMulSat(func: *Func, inst: Air.Inst.Index) !void { - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airMulSat", .{}); - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airShlSat(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (func.liveness.isUnused(inst)) - .unreach - else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu)) - return func.fail("TODO implement vector airShlSat with scalar rhs", .{}) - else - return func.fail("TODO implement airShlSat", .{}); - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airOptionalPayload(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = result: { - const pl_ty = func.typeOfIndex(inst); - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const opt_mcv = try func.resolveInst(ty_op.operand); - if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) { - switch (opt_mcv) { - .register => |pl_reg| try func.truncateRegister(pl_ty, pl_reg), - else => {}, - } - break :result opt_mcv; - } - - const pl_mcv = try func.allocRegOrMem(pl_ty, inst, true); - try func.genCopy(pl_ty, pl_mcv, opt_mcv); - break :result pl_mcv; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayloadPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement .optional_payload_ptr for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; - - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const dst_ty = func.typeOfIndex(inst); - const src_ty = func.typeOf(ty_op.operand); - const opt_ty = src_ty.childType(zcu); - const src_mcv = try func.resolveInst(ty_op.operand); - - if (opt_ty.optionalReprIsPayload(zcu)) { - break :result if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try func.copyToNewRegister(inst, src_mcv); - } - - const dst_mcv: MCValue = if (src_mcv.isRegister() and - func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try func.copyToNewRegister(inst, src_mcv); - - const pl_ty = dst_ty.childType(zcu); - const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); - try func.genSetMem( - .{ .reg = dst_mcv.getReg().? }, - pl_abi_size, - Type.bool, - .{ .immediate = 1 }, - ); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airUnwrapErrErr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const pt = func.pt; - const zcu = pt.zcu; - const err_union_ty = func.typeOf(ty_op.operand); - const err_ty = err_union_ty.errorUnionSet(zcu); - const payload_ty = err_union_ty.errorUnionPayload(zcu); - const operand = try func.resolveInst(ty_op.operand); - - const result: MCValue = result: { - if (err_ty.errorSetIsEmpty(zcu)) { - break :result .{ .immediate = 0 }; - } - - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :result operand; - } - - const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu)); - - switch (operand) { - .register => |reg| { - const eu_lock = func.register_manager.lockReg(reg); - defer if (eu_lock) |lock| func.register_manager.unlockReg(lock); - - const result = try func.copyToNewRegister(inst, operand); - if (err_off > 0) { - try func.genBinOp( - .shr, - result, - err_union_ty, - .{ .immediate = @as(u6, @intCast(err_off * 8)) }, - Type.u8, - result.register, - ); - } - break :result result; - }, - .load_frame => |frame_addr| break :result .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + @as(i32, @intCast(err_off)), - } }, - else => return func.fail("TODO implement unwrap_err_err for {}", .{operand}), - } - }; - - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airUnwrapErrPayload(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand_ty = func.typeOf(ty_op.operand); - const operand = try func.resolveInst(ty_op.operand); - const result = try func.genUnwrapErrUnionPayloadMir(operand_ty, operand); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn genUnwrapErrUnionPayloadMir( - func: *Func, - err_union_ty: Type, - err_union: MCValue, -) !MCValue { - const pt = func.pt; - const zcu = pt.zcu; - const payload_ty = err_union_ty.errorUnionPayload(zcu); - - const result: MCValue = result: { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); - switch (err_union) { - .load_frame => |frame_addr| break :result .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + payload_off, - } }, - .register => |reg| { - const eu_lock = func.register_manager.lockReg(reg); - defer if (eu_lock) |lock| func.register_manager.unlockReg(lock); - - const result_reg = try func.copyToTmpRegister(err_union_ty, err_union); - if (payload_off > 0) { - try func.genBinOp( - .shr, - .{ .register = result_reg }, - err_union_ty, - .{ .immediate = @as(u6, @intCast(payload_off * 8)) }, - Type.u8, - result_reg, - ); - } - break :result .{ .register = result_reg }; - }, - else => return func.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}), - } - }; - - return result; -} - -// *(E!T) -> E -fn airUnwrapErrErrPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union error ptr for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -// *(E!T) -> *T -fn airUnwrapErrPayloadPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union payload ptr for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -// *(E!T) => *T -fn airErrUnionPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const zcu = func.pt.zcu; - const src_ty = func.typeOf(ty_op.operand); - const src_mcv = try func.resolveInst(ty_op.operand); - - // `src_reg` contains the pointer to the error union - const src_reg = switch (src_mcv) { - .register => |reg| reg, - else => try func.copyToTmpRegister(src_ty, src_mcv), - }; - const src_lock = func.register_manager.lockRegAssumeUnused(src_reg); - defer func.register_manager.unlockReg(src_lock); - - // we set the place of where the error would have been to 0 - const eu_ty = src_ty.childType(zcu); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); - try func.genSetMem(.{ .reg = src_reg }, err_off, err_ty, .{ .immediate = 0 }); - - const dst_reg, const dst_lock = if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - .{ src_reg, null } - else - try func.allocReg(.int); - defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); - - // move the pointer to be at the payload - const pl_off = errUnionPayloadOffset(pl_ty, zcu); - try func.genBinOp( - .add, - .{ .register = src_reg }, - Type.u64, - .{ .immediate = pl_off }, - Type.u64, - dst_reg, - ); - - break :result .{ .register = dst_reg }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void { - const result: MCValue = if (func.liveness.isUnused(inst)) - .unreach - else - return func.fail("TODO implement airErrReturnTrace for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airSetErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement airSetErrReturnTrace for {}", .{func.target.cpu.arch}); -} - -fn airSaveErrReturnTraceIndex(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{func.target.cpu.arch}); -} - -fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = result: { - const pl_ty = func.typeOf(ty_op.operand); - if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 }; - - const opt_ty = func.typeOfIndex(inst); - const pl_mcv = try func.resolveInst(ty_op.operand); - const same_repr = opt_ty.optionalReprIsPayload(zcu); - if (same_repr and func.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv; - - const pl_lock: ?RegisterLock = switch (pl_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (pl_lock) |lock| func.register_manager.unlockReg(lock); - - const opt_mcv = try func.allocRegOrMem(opt_ty, inst, false); - try func.genCopy(pl_ty, opt_mcv, pl_mcv); - - if (!same_repr) { - const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); - switch (opt_mcv) { - .load_frame => |frame_addr| { - try func.genCopy(pl_ty, opt_mcv, pl_mcv); - try func.genSetMem( - .{ .frame = frame_addr.index }, - frame_addr.off + pl_abi_size, - Type.u8, - .{ .immediate = 1 }, - ); - }, - else => unreachable, - } - } - break :result opt_mcv; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -/// T to E!T -fn airWrapErrUnionPayload(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const eu_ty = ty_op.ty.toType(); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - const operand = try func.resolveInst(ty_op.operand); - - const result: MCValue = result: { - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 }; - - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu)); - const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); - try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand); - try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }); - break :result .{ .load_frame = .{ .index = frame_index } }; - }; - - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -/// E to E!T -fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const eu_ty = ty_op.ty.toType(); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - - const result: MCValue = result: { - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try func.resolveInst(ty_op.operand); - - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu)); - const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); - try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null }); - const operand = try func.resolveInst(ty_op.operand); - try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand); - break :result .{ .load_frame = .{ .index = frame_index } }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airRuntimeNavPtr(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; - const ip = &zcu.intern_pool; - const ty_nav = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; - const ptr_ty: Type = .fromInterned(ty_nav.ty); - - const nav = ip.getNav(ty_nav.nav); - const tlv_sym_index = if (func.bin_file.cast(.elf)) |elf_file| sym: { - const zo = elf_file.zigObjectPtr().?; - if (nav.getExtern(ip)) |e| { - break :sym try elf_file.getGlobalSymbol(nav.name.toSlice(ip), e.lib_name.toSlice(ip)); - } - break :sym try zo.getOrCreateMetadataForNav(zcu, ty_nav.nav); - } else return func.fail("TODO runtime_nav_ptr on {}", .{func.bin_file.tag}); - - const dest_mcv = try func.allocRegOrMem(ptr_ty, inst, true); - if (dest_mcv.isRegister()) { - _ = try func.addInst(.{ - .tag = .pseudo_load_tlv, - .data = .{ .reloc = .{ - .register = dest_mcv.getReg().?, - .atom_index = try func.owner.getSymbolIndex(func), - .sym_index = tlv_sym_index, - } }, - }); - } else { - const tmp_reg, const tmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(tmp_lock); - _ = try func.addInst(.{ - .tag = .pseudo_load_tlv, - .data = .{ .reloc = .{ - .register = tmp_reg, - .atom_index = try func.owner.getSymbolIndex(func), - .sym_index = tlv_sym_index, - } }, - }); - try func.genCopy(ptr_ty, dest_mcv, .{ .register = tmp_reg }); - } - - return func.finishAir(inst, dest_mcv, .{ .none, .none, .none }); -} - -fn airTry(func: *Func, inst: Air.Inst.Index) !void { - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = func.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]); - const operand_ty = func.typeOf(pl_op.operand); - const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false); - return func.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn genTry( - func: *Func, - inst: Air.Inst.Index, - operand: Air.Inst.Ref, - body: []const Air.Inst.Index, - operand_ty: Type, - operand_is_ptr: bool, -) !MCValue { - _ = operand_is_ptr; - - const liveness_cond_br = func.liveness.getCondBr(inst); - - const operand_mcv = try func.resolveInst(operand); - const is_err_mcv = try func.isErr(null, operand_ty, operand_mcv); - - // A branch to the false section. Uses beq. 1 is the default "true" state. - const reloc = try func.condBr(Type.anyerror, is_err_mcv); - - if (func.liveness.operandDies(inst, 0)) { - if (operand.toIndex()) |operand_inst| try func.processDeath(operand_inst); - } - - func.scope_generation += 1; - const state = try func.saveState(); - - for (liveness_cond_br.else_deaths) |death| try func.processDeath(death); - try func.genBody(body); - try func.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - func.performReloc(reloc); - - for (liveness_cond_br.then_deaths) |death| try func.processDeath(death); - - const result = if (func.liveness.isUnused(inst)) - .unreach - else - try func.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv); - - return result; -} - -fn airSlicePtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = result: { - const src_mcv = try func.resolveInst(ty_op.operand); - if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; - - const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - const dst_ty = func.typeOfIndex(inst); - try func.genCopy(dst_ty, dst_mcv, src_mcv); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSliceLen(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const src_mcv = try func.resolveInst(ty_op.operand); - const ty = func.typeOfIndex(inst); - - switch (src_mcv) { - .load_frame => |frame_addr| { - const len_mcv: MCValue = .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + 8, - } }; - if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv; - - const dst_mcv = try func.allocRegOrMem(ty, inst, true); - try func.genCopy(Type.u64, dst_mcv, len_mcv); - break :result dst_mcv; - }, - .register_pair => |pair| { - const len_mcv: MCValue = .{ .register = pair[1] }; - - if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv; - - const dst_mcv = try func.allocRegOrMem(ty, inst, true); - try func.genCopy(Type.u64, dst_mcv, len_mcv); - break :result dst_mcv; - }, - else => return func.fail("TODO airSliceLen for {}", .{src_mcv}), - } - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPtrSliceLenPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const src_mcv = try func.resolveInst(ty_op.operand); - - const dst_reg, const dst_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(dst_lock); - const dst_mcv: MCValue = .{ .register = dst_reg }; - - try func.genCopy(Type.u64, dst_mcv, src_mcv.offset(8)); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPtrSlicePtrPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const opt_mcv = try func.resolveInst(ty_op.operand); - const dst_mcv = if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) - opt_mcv - else - try func.copyToNewRegister(inst, opt_mcv); - return func.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airSliceElemVal(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const result: MCValue = result: { - const elem_ty = func.typeOfIndex(inst); - if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const slice_ty = func.typeOf(bin_op.lhs); - const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); - const elem_ptr = try func.genSliceElemPtr(bin_op.lhs, bin_op.rhs); - const dst_mcv = try func.allocRegOrMem(elem_ty, inst, false); - try func.load(dst_mcv, elem_ptr, slice_ptr_field_type); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSliceElemPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; - const dst_mcv = try func.genSliceElemPtr(extra.lhs, extra.rhs); - return func.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); -} - -fn genSliceElemPtr(func: *Func, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { - const pt = func.pt; - const zcu = pt.zcu; - const slice_ty = func.typeOf(lhs); - const slice_mcv = try func.resolveInst(lhs); - const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (slice_mcv_lock) |lock| func.register_manager.unlockReg(lock); - - const elem_ty = slice_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - const index_ty = func.typeOf(rhs); - const index_mcv = try func.resolveInst(rhs); - const index_mcv_lock: ?RegisterLock = switch (index_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_mcv_lock) |lock| func.register_manager.unlockReg(lock); - - const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_size); - const offset_reg_lock = func.register_manager.lockRegAssumeUnused(offset_reg); - defer func.register_manager.unlockReg(offset_reg_lock); - - const addr_reg, const addr_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(addr_lock); - try func.genSetReg(Type.u64, addr_reg, slice_mcv); - - _ = try func.addInst(.{ - .tag = .add, - .data = .{ .r_type = .{ - .rd = addr_reg, - .rs1 = addr_reg, - .rs2 = offset_reg, - } }, - }); - - return .{ .register = addr_reg }; -} - -fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const result_ty = func.typeOfIndex(inst); - - const array_ty = func.typeOf(bin_op.lhs); - const array_mcv = try func.resolveInst(bin_op.lhs); - - const index_mcv = try func.resolveInst(bin_op.rhs); - const index_ty = func.typeOf(bin_op.rhs); - - const elem_ty = array_ty.childType(zcu); - const elem_abi_size = elem_ty.abiSize(zcu); - - const addr_reg, const addr_reg_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(addr_reg_lock); - - switch (array_mcv) { - .register => { - const frame_index = try func.allocFrameIndex(FrameAlloc.initType(array_ty, zcu)); - try func.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv); - try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = .{ .index = frame_index } }); - }, - .load_frame => |frame_addr| { - try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = frame_addr }); - }, - else => try func.genSetReg(Type.u64, addr_reg, array_mcv.address()), - } - - const dst_mcv = try func.allocRegOrMem(result_ty, inst, false); - - if (array_ty.isVector(zcu)) { - // we need to load the vector, vslidedown to get the element we want - // and store that element in a load frame. - - const src_reg, const src_lock = try func.allocReg(.vector); - defer func.register_manager.unlockReg(src_lock); - - // load the vector into a temporary register - try func.genCopy(array_ty, .{ .register = src_reg }, .{ .indirect = .{ .reg = addr_reg } }); - - // we need to construct a 1xbitSize vector because of how lane splitting works in RISC-V - const single_ty = try pt.vectorType(.{ .child = elem_ty.toIntern(), .len = 1 }); - - // we can do a shortcut here where we don't need a vslicedown - // and can just copy to the frame index. - if (!(index_mcv == .immediate and index_mcv.immediate == 0)) { - const index_reg = try func.copyToTmpRegister(Type.u64, index_mcv); - - _ = try func.addInst(.{ - .tag = .vslidedownvx, - .data = .{ .r_type = .{ - .rd = src_reg, - .rs1 = index_reg, - .rs2 = src_reg, - } }, - }); - } - - try func.genCopy(single_ty, dst_mcv, .{ .register = src_reg }); - break :result dst_mcv; - } - - const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_abi_size); - const offset_lock = func.register_manager.lockRegAssumeUnused(offset_reg); - defer func.register_manager.unlockReg(offset_lock); - _ = try func.addInst(.{ - .tag = .add, - .data = .{ .r_type = .{ - .rd = addr_reg, - .rs1 = addr_reg, - .rs2 = offset_reg, - } }, - }); - - try func.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrElemVal(func: *Func, inst: Air.Inst.Index) !void { - const is_volatile = false; // TODO - const pt = func.pt; - const zcu = pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const base_ptr_ty = func.typeOf(bin_op.lhs); - - const result: MCValue = if (!is_volatile and func.liveness.isUnused(inst)) .unreach else result: { - const elem_ty = base_ptr_ty.elemType2(zcu); - if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const base_ptr_mcv = try func.resolveInst(bin_op.lhs); - const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); - - const index_mcv = try func.resolveInst(bin_op.rhs); - const index_lock: ?RegisterLock = switch (index_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_lock) |lock| func.register_manager.unlockReg(lock); - - const elem_ptr_reg, const elem_ptr_lock = if (base_ptr_mcv.isRegister() and - func.liveness.operandDies(inst, 0)) - .{ base_ptr_mcv.register, null } - else blk: { - const reg, const lock = try func.allocReg(.int); - try func.genSetReg(base_ptr_ty, reg, base_ptr_mcv); - break :blk .{ reg, lock }; - }; - defer if (elem_ptr_lock) |lock| func.register_manager.unlockReg(lock); - - try func.genBinOp( - .ptr_add, - base_ptr_mcv, - base_ptr_ty, - index_mcv, - Type.u64, - elem_ptr_reg, - ); - - const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - const dst_lock = switch (dst_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); - - try func.load(dst_mcv, .{ .register = elem_ptr_reg }, base_ptr_ty); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const elem_ptr_ty = func.typeOfIndex(inst); - const base_ptr_ty = func.typeOf(extra.lhs); - - if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { - @panic("audit"); - } - - const base_ptr_mcv = try func.resolveInst(extra.lhs); - const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); - - const index_mcv = try func.resolveInst(extra.rhs); - const index_lock: ?RegisterLock = switch (index_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_lock) |lock| func.register_manager.unlockReg(lock); - - const result_reg, const result_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(result_lock); - - try func.genBinOp( - .ptr_add, - base_ptr_mcv, - base_ptr_ty, - index_mcv, - Type.u64, - result_reg, - ); - - break :result MCValue{ .register = result_reg }; - }; - - return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airSetUnionTag(func: *Func, inst: Air.Inst.Index) !void { - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - _ = bin_op; - return func.fail("TODO implement airSetUnionTag for {}", .{func.target.cpu.arch}); - // return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const tag_ty = func.typeOfIndex(inst); - const union_ty = func.typeOf(ty_op.operand); - const layout = union_ty.unionGetLayout(zcu); - - if (layout.tag_size == 0) { - return func.finishAir(inst, .none, .{ ty_op.operand, .none, .none }); - } - - const operand = try func.resolveInst(ty_op.operand); - - const frame_mcv = try func.allocRegOrMem(union_ty, null, false); - try func.genCopy(union_ty, frame_mcv, operand); - - const tag_abi_size = tag_ty.abiSize(zcu); - const result_reg, const result_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(result_lock); - - switch (frame_mcv) { - .load_frame => { - if (tag_abi_size <= 8) { - const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align)) - @intCast(layout.payload_size) - else - 0; - - try func.genCopy( - tag_ty, - .{ .register = result_reg }, - frame_mcv.offset(off), - ); - } else { - return func.fail( - "TODO implement get_union_tag for ABI larger than 8 bytes and operand {}, tag {f}", - .{ frame_mcv, tag_ty.fmt(pt) }, - ); - } - }, - else => return func.fail("TODO: airGetUnionTag {s}", .{@tagName(operand)}), - } - - return func.finishAir(inst, .{ .register = result_reg }, .{ ty_op.operand, .none, .none }); -} - -fn airClz(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try func.resolveInst(ty_op.operand); - const ty = func.typeOf(ty_op.operand); - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const src_reg, const src_lock = try func.promoteReg(ty, operand); - defer if (src_lock) |lock| func.register_manager.unlockReg(lock); - - const dst_reg: Register = if (func.reuseOperand( - inst, - ty_op.operand, - 0, - operand, - ) and operand == .register) - operand.register - else - (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register; - - const bit_size = ty.bitSize(func.pt.zcu); - if (!math.isPowerOfTwo(bit_size)) try func.truncateRegister(ty, src_reg); - - if (bit_size > 64) { - return func.fail("TODO: airClz > 64 bits, found {d}", .{bit_size}); - } - - _ = try func.addInst(.{ - .tag = switch (bit_size) { - 32 => .clzw, - else => .clz, - }, - .data = .{ - .r_type = .{ - .rs2 = .zero, // rs2 is 0 filled in the spec - .rs1 = src_reg, - .rd = dst_reg, - }, - }, - }); - - if (!(bit_size == 32 or bit_size == 64)) { - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ .i_type = .{ - .rd = dst_reg, - .rs1 = dst_reg, - .imm12 = Immediate.s(-@as(i12, @intCast(64 - bit_size % 64))), - } }, - }); - } - - break :result .{ .register = dst_reg }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airCtz(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - _ = ty_op; - return func.fail("TODO: finish ctz", .{}); -} - -fn airPopcount(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const pt = func.pt; - const zcu = pt.zcu; - - const operand = try func.resolveInst(ty_op.operand); - const src_ty = func.typeOf(ty_op.operand); - const operand_reg, const operand_lock = try func.promoteReg(src_ty, operand); - defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); - - const dst_reg, const dst_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(dst_lock); - - const bit_size = src_ty.bitSize(zcu); - switch (bit_size) { - 32, 64 => {}, - 1...31, 33...63 => try func.truncateRegister(src_ty, operand_reg), - else => return func.fail("TODO: airPopcount > 64 bits", .{}), - } - - _ = try func.addInst(.{ - .tag = if (bit_size <= 32) .cpopw else .cpop, - .data = .{ - .r_type = .{ - .rd = dst_reg, - .rs1 = operand_reg, - .rs2 = @enumFromInt(0b00010), // this is the cpop funct5 - }, - }, - }); - - break :result .{ .register = dst_reg }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airAbs(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const ty = func.typeOf(ty_op.operand); - const scalar_ty = ty.scalarType(zcu); - const operand = try func.resolveInst(ty_op.operand); - - switch (scalar_ty.zigTypeTag(zcu)) { - .int => if (ty.zigTypeTag(zcu) == .vector) { - return func.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)}); - } else { - const int_info = scalar_ty.intInfo(zcu); - const int_bits = int_info.bits; - switch (int_bits) { - 32, 64 => {}, - else => return func.fail("TODO: airAbs Int size {d}", .{int_bits}), - } - - const return_mcv = try func.copyToNewRegister(inst, operand); - const operand_reg = return_mcv.register; - - const temp_reg, const temp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(temp_lock); - - _ = try func.addInst(.{ - .tag = switch (int_bits) { - 32 => .sraiw, - 64 => .srai, - else => unreachable, - }, - .data = .{ .i_type = .{ - .rd = temp_reg, - .rs1 = operand_reg, - .imm12 = Immediate.u(int_bits - 1), - } }, - }); - - _ = try func.addInst(.{ - .tag = .xor, - .data = .{ .r_type = .{ - .rd = operand_reg, - .rs1 = operand_reg, - .rs2 = temp_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = switch (int_bits) { - 32 => .subw, - 64 => .sub, - else => unreachable, - }, - .data = .{ .r_type = .{ - .rd = operand_reg, - .rs1 = operand_reg, - .rs2 = temp_reg, - } }, - }); - - break :result return_mcv; - }, - .float => { - const float_bits = scalar_ty.floatBits(zcu.getTarget()); - const mnem: Mnemonic = switch (float_bits) { - 16 => return func.fail("TODO: airAbs 16-bit float", .{}), - 32 => .fsgnjxs, - 64 => .fsgnjxd, - 80 => return func.fail("TODO: airAbs 80-bit float", .{}), - 128 => return func.fail("TODO: airAbs 128-bit float", .{}), - else => unreachable, - }; - - const return_mcv = try func.copyToNewRegister(inst, operand); - const operand_reg = return_mcv.register; - - assert(operand_reg.class() == .float); - - _ = try func.addInst(.{ - .tag = mnem, - .data = .{ - .r_type = .{ - .rd = operand_reg, - .rs1 = operand_reg, - .rs2 = operand_reg, - }, - }, - }); - - break :result return_mcv; - }, - else => return func.fail("TODO: implement airAbs {f}", .{scalar_ty.fmt(pt)}), - } - - break :result .unreach; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airByteSwap(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const pt = func.pt; - const zcu = pt.zcu; - const ty = func.typeOf(ty_op.operand); - const operand = try func.resolveInst(ty_op.operand); - - switch (ty.zigTypeTag(zcu)) { - .int => { - const int_bits = ty.intInfo(zcu).bits; - - // bytes are no-op - if (int_bits == 8 and func.reuseOperand(inst, ty_op.operand, 0, operand)) { - return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none }); - } - - const dest_mcv = try func.copyToNewRegister(inst, operand); - const dest_reg = dest_mcv.register; - - switch (int_bits) { - 16 => { - const temp_reg, const temp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(temp_lock); - - _ = try func.addInst(.{ - .tag = .srli, - .data = .{ .i_type = .{ - .imm12 = Immediate.s(8), - .rd = temp_reg, - .rs1 = dest_reg, - } }, - }); - - _ = try func.addInst(.{ - .tag = .slli, - .data = .{ .i_type = .{ - .imm12 = Immediate.s(8), - .rd = dest_reg, - .rs1 = dest_reg, - } }, - }); - _ = try func.addInst(.{ - .tag = .@"or", - .data = .{ .r_type = .{ - .rd = dest_reg, - .rs1 = dest_reg, - .rs2 = temp_reg, - } }, - }); - }, - else => return func.fail("TODO: {d} bits for airByteSwap", .{int_bits}), - } - - break :result dest_mcv; - }, - else => return func.fail("TODO: airByteSwap {f}", .{ty.fmt(pt)}), - } - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airBitReverse(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airBitReverse for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const pt = func.pt; - const zcu = pt.zcu; - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const ty = func.typeOf(un_op); - - const operand = try func.resolveInst(un_op); - const operand_bit_size = ty.bitSize(zcu); - - if (!math.isPowerOfTwo(operand_bit_size)) - return func.fail("TODO: airUnaryMath non-pow 2", .{}); - - const operand_reg, const operand_lock = try func.promoteReg(ty, operand); - defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); - - const dst_class = func.typeRegClass(ty); - const dst_reg, const dst_lock = try func.allocReg(dst_class); - defer func.register_manager.unlockReg(dst_lock); - - switch (ty.zigTypeTag(zcu)) { - .float => { - assert(dst_class == .float); - - switch (operand_bit_size) { - 16, 80, 128 => return func.fail("TODO: airUnaryMath Float bit-size {}", .{operand_bit_size}), - 32, 64 => {}, - else => unreachable, - } - - switch (tag) { - .sqrt => { - _ = try func.addInst(.{ - .tag = if (operand_bit_size == 64) .fsqrtd else .fsqrts, - .data = .{ - .r_type = .{ - .rd = dst_reg, - .rs1 = operand_reg, - .rs2 = .f0, // unused, spec says it's 0 - }, - }, - }); - }, - - else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}), - } - }, - .int => { - assert(dst_class == .int); - - switch (tag) { - else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}), - } - }, - else => return func.fail("TODO: airUnaryMath ty: {f}", .{ty.fmt(pt)}), - } - - break :result MCValue{ .register = dst_reg }; - }; - - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn reuseOperand( - func: *Func, - inst: Air.Inst.Index, - operand: Air.Inst.Ref, - op_index: Air.Liveness.OperandInt, - mcv: MCValue, -) bool { - return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); -} - -fn reuseOperandAdvanced( - func: *Func, - inst: Air.Inst.Index, - operand: Air.Inst.Ref, - op_index: Air.Liveness.OperandInt, - mcv: MCValue, - maybe_tracked_inst: ?Air.Inst.Index, -) bool { - if (!func.liveness.operandDies(inst, op_index)) - return false; - - switch (mcv) { - .register, - .register_pair, - => for (mcv.getRegs()) |reg| { - // If it's in the registers table, need to associate the register(s) with the - // new instruction. - if (maybe_tracked_inst) |tracked_inst| { - if (!func.register_manager.isRegFree(reg)) { - if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { - func.register_manager.registers[index] = tracked_inst; - } - } - } else func.register_manager.freeReg(reg); - }, - .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, - else => return false, - } - - // Prevent the operand deaths processing code from deallocating it. - func.reused_operands.set(op_index); - const op_inst = operand.toIndex().?; - func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst); - - return true; -} - -fn airLoad(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const elem_ty = func.typeOfIndex(inst); - - const result: MCValue = result: { - if (!elem_ty.hasRuntimeBits(zcu)) - break :result .none; - - const ptr = try func.resolveInst(ty_op.operand); - const is_volatile = func.typeOf(ty_op.operand).isVolatilePtr(zcu); - if (func.liveness.isUnused(inst) and !is_volatile) - break :result .unreach; - - const elem_size = elem_ty.abiSize(zcu); - - const dst_mcv: MCValue = blk: { - // The MCValue that holds the pointer can be re-used as the value. - // - "ptr" is 8 bytes, and if the element is more than that, we cannot reuse it. - // - // - "ptr" will be stored in an integer register, so the type that we're gonna - // load into it must also be a type that can be inside of an integer register - if (elem_size <= 8 and - (if (ptr == .register) func.typeRegClass(elem_ty) == ptr.register.class() else true) and - func.reuseOperand(inst, ty_op.operand, 0, ptr)) - { - break :blk ptr; - } else { - break :blk try func.allocRegOrMem(elem_ty, inst, true); - } - }; - - try func.load(dst_mcv, ptr, func.typeOf(ty_op.operand)); - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn load(func: *Func, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void { - const pt = func.pt; - const zcu = pt.zcu; - const dst_ty = ptr_ty.childType(zcu); - - log.debug("loading {}:{f} into {}", .{ ptr_mcv, ptr_ty.fmt(pt), dst_mcv }); - - switch (ptr_mcv) { - .none, - .undef, - .unreach, - .dead, - .register_pair, - .reserved_frame, - => unreachable, // not a valid pointer - - .immediate, - .register, - .register_offset, - .lea_frame, - .lea_symbol, - => try func.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()), - - .memory, - .indirect, - .load_symbol, - .load_frame, - => { - const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); - const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); - defer func.register_manager.unlockReg(addr_lock); - - try func.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); - }, - .air_ref => |ptr_ref| try func.load(dst_mcv, try func.resolveInst(ptr_ref), ptr_ty), - } -} - -fn airStore(func: *Func, inst: Air.Inst.Index, safety: bool) !void { - if (safety) { - // TODO if the value is undef, write 0xaa bytes to dest - } else { - // TODO if the value is undef, don't lower this instruction - } - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ptr = try func.resolveInst(bin_op.lhs); - const value = try func.resolveInst(bin_op.rhs); - const ptr_ty = func.typeOf(bin_op.lhs); - - try func.store(ptr, value, ptr_ty); - - return func.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -/// Loads `value` into the "payload" of `pointer`. -fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void { - const zcu = func.pt.zcu; - const src_ty = ptr_ty.childType(zcu); - log.debug("storing {}:{f} in {}:{f}", .{ src_mcv, src_ty.fmt(func.pt), ptr_mcv, ptr_ty.fmt(func.pt) }); - - switch (ptr_mcv) { - .none => unreachable, - .undef => unreachable, - .unreach => unreachable, - .dead => unreachable, - .register_pair => unreachable, - .reserved_frame => unreachable, - - .immediate, - .register, - .register_offset, - .lea_symbol, - .lea_frame, - => try func.genCopy(src_ty, ptr_mcv.deref(), src_mcv), - - .memory, - .indirect, - .load_symbol, - .load_frame, - => { - const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); - const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); - defer func.register_manager.unlockReg(addr_lock); - - try func.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); - }, - .air_ref => |ptr_ref| try func.store(try func.resolveInst(ptr_ref), src_mcv, ptr_ty), - } -} - -fn airStructFieldPtr(func: *Func, inst: Air.Inst.Index) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.StructField, ty_pl.payload).data; - const result = try func.structFieldPtr(inst, extra.struct_operand, extra.field_index); - return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); -} - -fn airStructFieldPtrIndex(func: *Func, inst: Air.Inst.Index, index: u8) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = try func.structFieldPtr(inst, ty_op.operand, index); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn structFieldPtr(func: *Func, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue { - const pt = func.pt; - const zcu = pt.zcu; - const ptr_field_ty = func.typeOfIndex(inst); - const ptr_container_ty = func.typeOf(operand); - const container_ty = ptr_container_ty.childType(zcu); - - const field_offset: i32 = switch (container_ty.containerLayout(zcu)) { - .auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)), - .@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) + - (if (zcu.typeToStruct(container_ty)) |struct_obj| zcu.structPackedFieldBitOffset(struct_obj, index) else 0) - - ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8), - }; - - const src_mcv = try func.resolveInst(operand); - const dst_mcv = if (switch (src_mcv) { - .immediate, .lea_frame => true, - .register, .register_offset => func.reuseOperand(inst, operand, 0, src_mcv), - else => false, - }) src_mcv else try func.copyToNewRegister(inst, src_mcv); - return dst_mcv.offset(field_offset); -} - -fn airStructFieldVal(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.StructField, ty_pl.payload).data; - const operand = extra.struct_operand; - const index = extra.field_index; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const src_mcv = try func.resolveInst(operand); - const struct_ty = func.typeOf(operand); - const field_ty = struct_ty.fieldType(index, zcu); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const field_off: u32 = switch (struct_ty.containerLayout(zcu)) { - .auto, .@"extern" => @intCast(struct_ty.structFieldOffset(index, zcu) * 8), - .@"packed" => if (zcu.typeToStruct(struct_ty)) |struct_type| - zcu.structPackedFieldBitOffset(struct_type, index) - else - 0, - }; - - switch (src_mcv) { - .dead, .unreach => unreachable, - .register => |src_reg| { - const src_reg_lock = func.register_manager.lockRegAssumeUnused(src_reg); - defer func.register_manager.unlockReg(src_reg_lock); - - const dst_reg = if (field_off == 0) - (try func.copyToNewRegister(inst, src_mcv)).register - else - try func.copyToTmpRegister(Type.u64, .{ .register = src_reg }); - - const dst_mcv: MCValue = .{ .register = dst_reg }; - const dst_lock = func.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); - - if (field_off > 0) { - _ = try func.addInst(.{ - .tag = .srli, - .data = .{ .i_type = .{ - .imm12 = Immediate.u(@intCast(field_off)), - .rd = dst_reg, - .rs1 = dst_reg, - } }, - }); - } - - if (field_off == 0) { - try func.truncateRegister(field_ty, dst_reg); - } - - break :result if (field_off == 0) dst_mcv else try func.copyToNewRegister(inst, dst_mcv); - }, - .load_frame => { - const field_abi_size: u32 = @intCast(field_ty.abiSize(zcu)); - if (field_off % 8 == 0) { - const field_byte_off = @divExact(field_off, 8); - const off_mcv = src_mcv.address().offset(@intCast(field_byte_off)).deref(); - const field_bit_size = field_ty.bitSize(zcu); - - if (field_abi_size <= 8) { - const int_ty = try pt.intType( - if (field_ty.isAbiInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned, - @intCast(field_bit_size), - ); - - const dst_reg, const dst_lock = try func.allocReg(.int); - const dst_mcv = MCValue{ .register = dst_reg }; - defer func.register_manager.unlockReg(dst_lock); - - try func.genCopy(int_ty, dst_mcv, off_mcv); - break :result try func.copyToNewRegister(inst, dst_mcv); - } - - const container_abi_size: u32 = @intCast(struct_ty.abiSize(zcu)); - const dst_mcv = if (field_byte_off + field_abi_size <= container_abi_size and - func.reuseOperand(inst, operand, 0, src_mcv)) - off_mcv - else dst: { - const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - try func.genCopy(field_ty, dst_mcv, off_mcv); - break :dst dst_mcv; - }; - if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) { - const tmp_reg, const tmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(tmp_lock); - - const hi_mcv = - dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref(); - try func.genSetReg(Type.u64, tmp_reg, hi_mcv); - try func.genCopy(Type.u64, hi_mcv, .{ .register = tmp_reg }); - } - break :result dst_mcv; - } - - return func.fail("TODO: airStructFieldVal load_frame field_off non multiple of 8", .{}); - }, - else => return func.fail("TODO: airStructField {s}", .{@tagName(src_mcv)}), - } - }; - - return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); -} - -fn airFieldParentPtr(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement codegen airFieldParentPtr", .{}); -} - -fn genArgDbgInfo(func: *const Func, name: []const u8, ty: Type, mcv: MCValue) InnerError!void { - assert(!func.mod.strip); - - // TODO: Add a pseudo-instruction or something to defer this work until Emit. - // We aren't allowed to interact with linker state here. - if (true) return; - switch (func.debug_output) { - .dwarf => |dw| switch (mcv) { - .register => |reg| dw.genLocalDebugInfo( - .local_arg, - name, - ty, - .{ .reg = reg.dwarfNum() }, - ) catch |err| return func.fail("failed to generate debug info: {s}", .{@errorName(err)}), - .load_frame => {}, - else => {}, - }, - .plan9 => {}, - .none => {}, - } -} - -fn airArg(func: *Func, inst: Air.Inst.Index) InnerError!void { - const zcu = func.pt.zcu; - - var arg_index = func.arg_index; - - // we skip over args that have no bits - while (func.args[arg_index] == .none) arg_index += 1; - func.arg_index = arg_index + 1; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const src_mcv = func.args[arg_index]; - const arg_ty = func.typeOfIndex(inst); - - const dst_mcv = try func.allocRegOrMem(arg_ty, inst, false); - - log.debug("airArg {} -> {}", .{ src_mcv, dst_mcv }); - - try func.genCopy(arg_ty, dst_mcv, src_mcv); - - const arg = func.air.instructions.items(.data)[@intFromEnum(inst)].arg; - // can delete `func.func_index` if this logic is moved to emit - const func_zir = zcu.funcInfo(func.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?; - const file = zcu.fileByIndex(func_zir.file); - const zir = &file.zir.?; - const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); - - try func.genArgDbgInfo(name, arg_ty, src_mcv); - break :result dst_mcv; - }; - - return func.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airTrap(func: *Func) !void { - _ = try func.addInst(.{ - .tag = .unimp, - .data = .none, - }); - return func.finishAirBookkeeping(); -} - -fn airBreakpoint(func: *Func) !void { - _ = try func.addInst(.{ - .tag = .ebreak, - .data = .none, - }); - return func.finishAirBookkeeping(); -} - -fn airRetAddr(func: *Func, inst: Air.Inst.Index) !void { - const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - try func.genCopy(Type.u64, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }); - return func.finishAir(inst, dst_mcv, .{ .none, .none, .none }); -} - -fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void { - const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - try func.genCopy(Type.u64, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }); - return func.finishAir(inst, dst_mcv, .{ .none, .none, .none }); -} - -fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void { - if (modifier == .always_tail) return func.fail("TODO implement tail calls for riscv64", .{}); - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const callee = pl_op.operand; - const extra = func.air.extraData(Air.Call, pl_op.payload); - const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]); - - const expected_num_args = 8; - const ExpectedContents = extern struct { - vals: [expected_num_args][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)), - }; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), func.gpa); - const allocator = stack.get(); - - const arg_tys = try allocator.alloc(Type, arg_refs.len); - defer allocator.free(arg_tys); - for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = func.typeOf(arg_ref); - - const arg_vals = try allocator.alloc(MCValue, arg_refs.len); - defer allocator.free(arg_vals); - for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref }; - - const call_ret = try func.genCall(.{ .air = callee }, arg_tys, arg_vals); - - var bt = func.liveness.iterateBigTomb(inst); - try func.feed(&bt, pl_op.operand); - for (arg_refs) |arg_ref| try func.feed(&bt, arg_ref); - - const result = if (func.liveness.isUnused(inst)) .unreach else call_ret; - return func.finishAirResult(inst, result); -} - -fn genCall( - func: *Func, - info: union(enum) { - air: Air.Inst.Ref, - lib: struct { - return_type: InternPool.Index, - param_types: []const InternPool.Index, - lib: ?[]const u8 = null, - callee: []const u8, - }, - }, - arg_tys: []const Type, - args: []const MCValue, -) !MCValue { - const pt = func.pt; - const zcu = pt.zcu; - - const fn_ty = switch (info) { - .air => |callee| fn_info: { - const callee_ty = func.typeOf(callee); - break :fn_info switch (callee_ty.zigTypeTag(zcu)) { - .@"fn" => callee_ty, - .pointer => callee_ty.childType(zcu), - else => unreachable, - }; - }, - .lib => |lib| try pt.funcType(.{ - .param_types = lib.param_types, - .return_type = lib.return_type, - .cc = func.target.cCallingConvention().?, - }), - }; - - const fn_info = zcu.typeToFunc(fn_ty).?; - - const allocator = func.gpa; - - const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len); - defer allocator.free(var_args); - for (var_args, arg_tys[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; - - var call_info = try func.resolveCallingConventionValues(fn_info, var_args); - defer call_info.deinit(func); - - // We need a properly aligned and sized call frame to be able to call this function. - { - const needed_call_frame = FrameAlloc.init(.{ - .size = call_info.stack_byte_count, - .alignment = call_info.stack_align, - }); - const frame_allocs_slice = func.frame_allocs.slice(); - const stack_frame_size = - &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; - stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size); - const stack_frame_align = - &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; - stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align); - } - - var reg_locks = std.array_list.Managed(?RegisterLock).init(allocator); - defer reg_locks.deinit(); - try reg_locks.ensureTotalCapacity(8); - defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| func.register_manager.unlockReg(lock); - - const frame_indices = try allocator.alloc(FrameIndex, args.len); - defer allocator.free(frame_indices); - - switch (call_info.return_value.long) { - .none, .unreach => {}, - .indirect => |reg_off| try func.register_manager.getReg(reg_off.reg, null), - else => unreachable, - } - for (call_info.args, args, arg_tys, frame_indices) |dst_arg, src_arg, arg_ty, *frame_index| { - switch (dst_arg) { - .none => {}, - .register => |reg| { - try func.register_manager.getReg(reg, null); - try reg_locks.append(func.register_manager.lockReg(reg)); - }, - .register_pair => |regs| { - for (regs) |reg| try func.register_manager.getReg(reg, null); - try reg_locks.appendSlice(&func.register_manager.lockRegs(2, regs)); - }, - .indirect => |reg_off| { - frame_index.* = try func.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu)); - try func.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg); - try func.register_manager.getReg(reg_off.reg, null); - try reg_locks.append(func.register_manager.lockReg(reg_off.reg)); - }, - else => return func.fail("TODO: genCall set arg {s}", .{@tagName(dst_arg)}), - } - } - - switch (call_info.return_value.long) { - .none, .unreach => {}, - .indirect => |reg_off| { - const ret_ty = Type.fromInterned(fn_info.return_type); - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu)); - try func.genSetReg(Type.u64, reg_off.reg, .{ - .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, - }); - call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } }; - try reg_locks.append(func.register_manager.lockReg(reg_off.reg)); - }, - else => unreachable, - } - - for (call_info.args, arg_tys, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| { - switch (dst_arg) { - .none, .load_frame => {}, - .register_pair => try func.genCopy(arg_ty, dst_arg, src_arg), - .register => |dst_reg| try func.genSetReg( - arg_ty, - dst_reg, - src_arg, - ), - .indirect => |reg_off| try func.genSetReg(Type.u64, reg_off.reg, .{ - .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, - }), - else => return func.fail("TODO: genCall actual set {s}", .{@tagName(dst_arg)}), - } - } - - // Due to incremental compilation, how function calls are generated depends - // on linking. - switch (info) { - .air => |callee| { - if (try func.air.value(callee, pt)) |func_value| { - const func_key = zcu.intern_pool.indexToKey(func_value.ip_index); - switch (switch (func_key) { - else => func_key, - .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { - .nav => |nav| zcu.intern_pool.indexToKey(zcu.navValue(nav).toIntern()), - else => func_key, - } else func_key, - }) { - .func => |func_val| { - if (func.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(zcu, func_val.owner_nav); - - if (func.mod.pic) { - return func.fail("TODO: genCall pic", .{}); - } else { - try func.genSetReg(Type.u64, .ra, .{ .lea_symbol = .{ .sym = sym_index } }); - _ = try func.addInst(.{ - .tag = .jalr, - .data = .{ .i_type = .{ - .rd = .ra, - .rs1 = .ra, - .imm12 = Immediate.s(0), - } }, - }); - } - } else unreachable; // not a valid riscv64 format - }, - .@"extern" => |@"extern"| { - const lib_name = @"extern".lib_name.toSlice(&zcu.intern_pool); - const name = @"extern".name.toSlice(&zcu.intern_pool); - const atom_index = try func.owner.getSymbolIndex(func); - - const elf_file = func.bin_file.cast(.elf).?; - _ = try func.addInst(.{ - .tag = .pseudo_extern_fn_reloc, - .data = .{ .reloc = .{ - .register = .ra, - .atom_index = atom_index, - .sym_index = try elf_file.getGlobalSymbol(name, lib_name), - } }, - }); - }, - else => return func.fail("TODO implement calling bitcasted functions", .{}), - } - } else { - assert(func.typeOf(callee).zigTypeTag(zcu) == .pointer); - const addr_reg, const addr_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(addr_lock); - try func.genSetReg(Type.u64, addr_reg, .{ .air_ref = callee }); - - _ = try func.addInst(.{ - .tag = .jalr, - .data = .{ .i_type = .{ - .rd = .ra, - .rs1 = addr_reg, - .imm12 = Immediate.s(0), - } }, - }); - } - }, - .lib => return func.fail("TODO: lib func calls", .{}), - } - - // reset the vector settings as they might have changed in the function - func.avl = null; - func.vtype = null; - - return call_info.return_value.short; -} - -fn airRet(func: *Func, inst: Air.Inst.Index, safety: bool) !void { - const pt = func.pt; - const zcu = pt.zcu; - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - - if (safety) { - // safe - } else { - // not safe - } - - const ret_ty = func.fn_type.fnReturnType(zcu); - switch (func.ret_mcv.short) { - .none => {}, - .register, - .register_pair, - => { - if (ret_ty.isVector(zcu)) { - const bit_size = ret_ty.totalVectorBits(zcu); - - // set the vtype to hold the entire vector's contents in a single element - try func.setVl(.zero, 0, .{ - .vsew = switch (bit_size) { - 8 => .@"8", - 16 => .@"16", - 32 => .@"32", - 64 => .@"64", - else => unreachable, - }, - .vlmul = .m1, - .vma = true, - .vta = true, - }); - } - - try func.genCopy(ret_ty, func.ret_mcv.short, .{ .air_ref = un_op }); - }, - .indirect => |reg_off| { - try func.register_manager.getReg(reg_off.reg, null); - const lock = func.register_manager.lockRegAssumeUnused(reg_off.reg); - defer func.register_manager.unlockReg(lock); - - try func.genSetReg(Type.u64, reg_off.reg, func.ret_mcv.long); - try func.genSetMem( - .{ .reg = reg_off.reg }, - reg_off.off, - ret_ty, - .{ .air_ref = un_op }, - ); - }, - else => unreachable, - } - - func.ret_mcv.liveOut(func, inst); - try func.finishAir(inst, .unreach, .{ un_op, .none, .none }); - - // Just add space for an instruction, reloced this later - const index = try func.addInst(.{ - .tag = .pseudo_j, - .data = .{ .j_type = .{ - .rd = .zero, - .inst = undefined, - } }, - }); - - try func.exitlude_jump_relocs.append(func.gpa, index); -} - -fn airRetLoad(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ptr = try func.resolveInst(un_op); - - const ptr_ty = func.typeOf(un_op); - switch (func.ret_mcv.short) { - .none => {}, - .register, .register_pair => try func.load(func.ret_mcv.short, ptr, ptr_ty), - .indirect => |reg_off| try func.genSetReg(ptr_ty, reg_off.reg, ptr), - else => unreachable, - } - func.ret_mcv.liveOut(func, inst); - try func.finishAir(inst, .unreach, .{ un_op, .none, .none }); - - // Just add space for an instruction, reloced this later - const index = try func.addInst(.{ - .tag = .pseudo_j, - .data = .{ .j_type = .{ - .rd = .zero, - .inst = undefined, - } }, - }); - - try func.exitlude_jump_relocs.append(func.gpa, index); -} - -fn airCmp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const pt = func.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const lhs_ty = func.typeOf(bin_op.lhs); - - switch (lhs_ty.zigTypeTag(zcu)) { - .int, - .@"enum", - .bool, - .pointer, - .error_set, - .optional, - .@"struct", - => { - const int_ty = switch (lhs_ty.zigTypeTag(zcu)) { - .@"enum" => lhs_ty.intTagType(zcu), - .int => lhs_ty, - .bool => Type.u1, - .pointer => Type.u64, - .error_set => Type.anyerror, - .optional => blk: { - const payload_ty = lhs_ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :blk Type.u1; - } else if (lhs_ty.isPtrLikeOptional(zcu)) { - break :blk Type.u64; - } else { - return func.fail("TODO riscv cmp non-pointer optionals", .{}); - } - }, - .@"struct" => blk: { - const struct_obj = ip.loadStructType(lhs_ty.toIntern()); - assert(struct_obj.layout == .@"packed"); - const backing_index = struct_obj.backingIntTypeUnordered(ip); - break :blk Type.fromInterned(backing_index); - }, - else => unreachable, - }; - - const int_info = int_ty.intInfo(zcu); - if (int_info.bits <= 64) { - break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); - } else { - return func.fail("TODO riscv cmp for ints > 64 bits", .{}); - } - }, - .float => { - const float_bits = lhs_ty.floatBits(func.target); - const float_reg_size: u32 = if (func.hasFeature(.d)) 64 else 32; - if (float_bits > float_reg_size) { - return func.fail("TODO: airCmp float > 64/32 bits", .{}); - } - break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); - }, - else => unreachable, - } - }; - - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airCmpVector(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement airCmpVector for {}", .{func.target.cpu.arch}); -} - -fn airCmpLtErrorsLen(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try func.resolveInst(un_op); - _ = operand; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLtErrorsLen for {}", .{func.target.cpu.arch}); - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void { - const dbg_stmt = func.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; - - _ = try func.addInst(.{ - .tag = .pseudo_dbg_line_column, - .data = .{ .pseudo_dbg_line_column = .{ - .line = dbg_stmt.line, - .column = dbg_stmt.column, - } }, - }); - - return func.finishAirBookkeeping(); -} - -fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload); - try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len])); -} - -fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void { - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const operand = pl_op.operand; - const ty = func.typeOf(operand); - const mcv = try func.resolveInst(operand); - const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); - - const tag = func.air.instructions.items(.tag)[@intFromEnum(inst)]; - func.genVarDbgInfo(tag, ty, mcv, name.toSlice(func.air)) catch |err| - return func.fail("failed to generate variable debug info: {s}", .{@errorName(err)}); - - return func.finishAir(inst, .unreach, .{ operand, .none, .none }); -} - -fn genVarDbgInfo( - func: Func, - tag: Air.Inst.Tag, - ty: Type, - mcv: MCValue, - name: []const u8, -) !void { - // TODO: Add a pseudo-instruction or something to defer this work until Emit. - // We aren't allowed to interact with linker state here. - if (true) return; - switch (func.debug_output) { - .dwarf => |dwarf| { - const loc: link.File.Dwarf.Loc = switch (mcv) { - .register => |reg| .{ .reg = reg.dwarfNum() }, - .memory => |address| .{ .constu = address }, - .immediate => |x| .{ .constu = x }, - .none => .empty, - else => blk: { - // log.warn("TODO generate debug info for {}", .{mcv}); - break :blk .empty; - }, - }; - try dwarf.genLocalDebugInfo(switch (tag) { - else => unreachable, - .dbg_var_ptr, .dbg_var_val => .local_var, - .dbg_arg_inline => .local_arg, - }, name, ty, loc); - }, - .plan9 => {}, - .none => {}, - } -} - -fn airCondBr(func: *Func, inst: Air.Inst.Index) !void { - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const cond = try func.resolveInst(pl_op.operand); - const cond_ty = func.typeOf(pl_op.operand); - const extra = func.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); - const liveness_cond_br = func.liveness.getCondBr(inst); - - // If the condition dies here in this condbr instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - if (func.liveness.operandDies(inst, 0)) { - if (pl_op.operand.toIndex()) |op_inst| try func.processDeath(op_inst); - } - - func.scope_generation += 1; - const state = try func.saveState(); - const reloc = try func.condBr(cond_ty, cond); - - for (liveness_cond_br.then_deaths) |death| try func.processDeath(death); - try func.genBody(then_body); - try func.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - func.performReloc(reloc); - - for (liveness_cond_br.else_deaths) |death| try func.processDeath(death); - try func.genBody(else_body); - try func.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - // We already took care of pl_op.operand earlier, so there's nothing left to do. - func.finishAirBookkeeping(); -} - -fn condBr(func: *Func, cond_ty: Type, condition: MCValue) !Mir.Inst.Index { - const cond_reg = try func.copyToTmpRegister(cond_ty, condition); - - return try func.addInst(.{ - .tag = .beq, - .data = .{ - .b_type = .{ - .rs1 = cond_reg, - .rs2 = .zero, - .inst = undefined, - }, - }, - }); -} - -fn isNull(func: *Func, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue { - const pt = func.pt; - const zcu = pt.zcu; - const pl_ty = opt_ty.optionalChild(zcu); - - const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu)) - .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty } - else - .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool }; - - const return_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - assert(return_mcv == .register); // should not be larger 8 bytes - const return_reg = return_mcv.register; - - switch (opt_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .register_offset, - .lea_frame, - .lea_symbol, - .reserved_frame, - .air_ref, - .register_pair, - => unreachable, - - .register => |opt_reg| { - if (some_info.off == 0) { - _ = try func.addInst(.{ - .tag = .pseudo_compare, - .data = .{ - .compare = .{ - .op = .eq, - .rd = return_reg, - .rs1 = opt_reg, - .rs2 = try func.copyToTmpRegister( - some_info.ty, - .{ .immediate = 0 }, - ), - .ty = Type.bool, - }, - }, - }); - return return_mcv; - } - assert(some_info.ty.ip_index == .bool_type); - const bit_offset: u7 = @intCast(some_info.off * 8); - - try func.genBinOp( - .shr, - .{ .register = opt_reg }, - Type.u64, - .{ .immediate = bit_offset }, - Type.u8, - return_reg, - ); - try func.truncateRegister(Type.u8, return_reg); - try func.genBinOp( - .cmp_eq, - .{ .register = return_reg }, - Type.u64, - .{ .immediate = 0 }, - Type.u8, - return_reg, - ); - - return return_mcv; - }, - - .load_frame => { - const opt_reg = try func.copyToTmpRegister( - some_info.ty, - opt_mcv.address().offset(some_info.off).deref(), - ); - const opt_reg_lock = func.register_manager.lockRegAssumeUnused(opt_reg); - defer func.register_manager.unlockReg(opt_reg_lock); - - _ = try func.addInst(.{ - .tag = .pseudo_compare, - .data = .{ - .compare = .{ - .op = .eq, - .rd = return_reg, - .rs1 = opt_reg, - .rs2 = try func.copyToTmpRegister( - some_info.ty, - .{ .immediate = 0 }, - ), - .ty = Type.bool, - }, - }, - }); - return return_mcv; - }, - - else => return func.fail("TODO: isNull {}", .{opt_mcv}), - } -} - -fn airIsNull(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try func.resolveInst(un_op); - const ty = func.typeOf(un_op); - const result = try func.isNull(inst, ty, operand); - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airIsNullPtr(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try func.resolveInst(un_op); - _ = operand; - const ty = func.typeOf(un_op); - _ = ty; - - if (true) return func.fail("TODO: airIsNullPtr", .{}); - - return func.finishAir(inst, .unreach, .{ un_op, .none, .none }); -} - -fn airIsNonNull(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try func.resolveInst(un_op); - const ty = func.typeOf(un_op); - const result = try func.isNull(inst, ty, operand); - assert(result == .register); - - _ = try func.addInst(.{ - .tag = .pseudo_not, - .data = .{ - .rr = .{ - .rd = result.register, - .rs = result.register, - }, - }, - }); - - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airIsNonNullPtr(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try func.resolveInst(un_op); - _ = operand; - const ty = func.typeOf(un_op); - _ = ty; - - if (true) return func.fail("TODO: airIsNonNullPtr", .{}); - - return func.finishAir(inst, .unreach, .{ un_op, .none, .none }); -} - -fn airIsErr(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const operand = try func.resolveInst(un_op); - const operand_ty = func.typeOf(un_op); - break :result try func.isErr(inst, operand_ty, operand); - }; - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airIsErrPtr(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const operand_ptr = try func.resolveInst(un_op); - const operand: MCValue = blk: { - if (func.reuseOperand(inst, un_op, 0, operand_ptr)) { - // The MCValue that holds the pointer can be re-used as the value. - break :blk operand_ptr; - } else { - break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - } - }; - try func.load(operand, operand_ptr, func.typeOf(un_op)); - const operand_ptr_ty = func.typeOf(un_op); - const operand_ty = operand_ptr_ty.childType(zcu); - - break :result try func.isErr(inst, operand_ty, operand); - }; - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -/// Generates a compare instruction which will indicate if `eu_mcv` is an error. -/// -/// Result is in the return register. -fn isErr(func: *Func, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue { - _ = maybe_inst; - const zcu = func.pt.zcu; - const err_ty = eu_ty.errorUnionSet(zcu); - if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false - const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); - - const return_reg, const return_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(return_lock); - - switch (eu_mcv) { - .register => |reg| { - const eu_lock = func.register_manager.lockReg(reg); - defer if (eu_lock) |lock| func.register_manager.unlockReg(lock); - - try func.genCopy(eu_ty, .{ .register = return_reg }, eu_mcv); - - if (err_off > 0) { - try func.genBinOp( - .shr, - .{ .register = return_reg }, - eu_ty, - .{ .immediate = @as(u6, @intCast(err_off * 8)) }, - Type.u8, - return_reg, - ); - } - - try func.genBinOp( - .cmp_neq, - .{ .register = return_reg }, - Type.anyerror, - .{ .immediate = 0 }, - Type.u8, - return_reg, - ); - }, - .load_frame => |frame_addr| { - try func.genBinOp( - .cmp_neq, - .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + err_off, - } }, - Type.anyerror, - .{ .immediate = 0 }, - Type.anyerror, - return_reg, - ); - }, - else => return func.fail("TODO implement isErr for {}", .{eu_mcv}), - } - - return .{ .register = return_reg }; -} - -fn airIsNonErr(func: *Func, inst: Air.Inst.Index) !void { - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const operand = try func.resolveInst(un_op); - const ty = func.typeOf(un_op); - break :result try func.isNonErr(inst, ty, operand); - }; - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn isNonErr(func: *Func, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue { - const is_err_res = try func.isErr(inst, eu_ty, eu_mcv); - switch (is_err_res) { - .register => |reg| { - _ = try func.addInst(.{ - .tag = .pseudo_not, - .data = .{ - .rr = .{ - .rd = reg, - .rs = reg, - }, - }, - }); - return is_err_res; - }, - // always false case - .immediate => |imm| { - assert(imm == 0); - return MCValue{ .immediate = @intFromBool(imm == 0) }; - }, - else => unreachable, - } -} - -fn airIsNonErrPtr(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const operand_ptr = try func.resolveInst(un_op); - const operand: MCValue = blk: { - if (func.reuseOperand(inst, un_op, 0, operand_ptr)) { - // The MCValue that holds the pointer can be re-used as the value. - break :blk operand_ptr; - } else { - break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); - } - }; - const operand_ptr_ty = func.typeOf(un_op); - const operand_ty = operand_ptr_ty.childType(zcu); - - try func.load(operand, operand_ptr, func.typeOf(un_op)); - break :result try func.isNonErr(inst, operand_ty, operand); - }; - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airLoop(func: *Func, inst: Air.Inst.Index) !void { - // A loop is a setup to be able to jump back to the beginning. - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const loop = func.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]); - - func.scope_generation += 1; - const state = try func.saveState(); - - try func.loops.putNoClobber(func.gpa, inst, .{ - .state = state, - .jmp_target = @intCast(func.mir_instructions.len), - }); - defer assert(func.loops.remove(inst)); - - try func.genBody(body); - - func.finishAirBookkeeping(); -} - -/// Send control flow to the `index` of `func.code`. -fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index { - return func.addInst(.{ - .tag = .pseudo_j, - .data = .{ .j_type = .{ - .rd = .zero, - .inst = index, - } }, - }); -} - -fn airBlock(func: *Func, inst: Air.Inst.Index) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Block, ty_pl.payload); - try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len])); -} - -fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { - // A block is a setup to be able to jump to the end. - const inst_tracking_i = func.inst_tracking.count(); - func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach)); - - func.scope_generation += 1; - try func.blocks.putNoClobber(func.gpa, inst, .{ .state = func.initRetroactiveState() }); - const liveness = func.liveness.getBlock(inst); - - // TODO emit debug info lexical block - try func.genBody(body); - - var block_data = func.blocks.fetchRemove(inst).?; - defer block_data.value.deinit(func.gpa); - if (block_data.value.relocs.items.len > 0) { - try func.restoreState(block_data.value.state, liveness.deaths, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - for (block_data.value.relocs.items) |reloc| func.performReloc(reloc); - } - - if (std.debug.runtime_safety) assert(func.inst_tracking.getIndex(inst).? == inst_tracking_i); - const tracking = &func.inst_tracking.values()[inst_tracking_i]; - if (func.liveness.isUnused(inst)) try tracking.die(func, inst); - func.getValueIfFree(tracking.short, inst); - func.finishAirBookkeeping(); -} - -fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { - const switch_br = func.air.unwrapSwitch(inst); - const condition = try func.resolveInst(switch_br.operand); - - // If the condition dies here in this switch instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - if (func.liveness.operandDies(inst, 0)) { - if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); - } - - try func.lowerSwitchBr(inst, switch_br, condition); - - // We already took care of pl_op.operand earlier, so there's nothing left to do - func.finishAirBookkeeping(); -} - -fn lowerSwitchBr( - func: *Func, - inst: Air.Inst.Index, - switch_br: Air.UnwrappedSwitch, - condition: MCValue, -) !void { - const condition_ty = func.typeOf(switch_br.operand); - const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1); - defer func.gpa.free(liveness.deaths); - - func.scope_generation += 1; - const state = try func.saveState(); - - var it = switch_br.iterateCases(); - while (it.next()) |case| { - var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); - defer func.gpa.free(relocs); - - for (case.items, relocs[0..case.items.len]) |item, *reloc| { - const item_mcv = try func.resolveInst(item); - - const cond_lock = switch (condition) { - .register => func.register_manager.lockRegAssumeUnused(condition.register), - else => null, - }; - defer if (cond_lock) |lock| func.register_manager.unlockReg(lock); - - const cmp_reg, const cmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(cmp_lock); - - try func.genBinOp( - .cmp_neq, - condition, - condition_ty, - item_mcv, - condition_ty, - cmp_reg, - ); - - reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg }); - } - - for (case.ranges, relocs[case.items.len..]) |range, *reloc| { - const min_mcv = try func.resolveInst(range[0]); - const max_mcv = try func.resolveInst(range[1]); - const cond_lock = switch (condition) { - .register => func.register_manager.lockRegAssumeUnused(condition.register), - else => null, - }; - defer if (cond_lock) |lock| func.register_manager.unlockReg(lock); - - const temp_cmp_reg, const temp_cmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(temp_cmp_lock); - - // is `condition` less than `min`? is "true", we've failed - try func.genBinOp( - .cmp_gte, - condition, - condition_ty, - min_mcv, - condition_ty, - temp_cmp_reg, - ); - - // if the compare was true, we will jump to the fail case and fall through - // to the next checks - const lt_fail_reloc = try func.condBr(condition_ty, .{ .register = temp_cmp_reg }); - try func.genBinOp( - .cmp_gt, - condition, - condition_ty, - max_mcv, - condition_ty, - temp_cmp_reg, - ); - - reloc.* = try func.condBr(condition_ty, .{ .register = temp_cmp_reg }); - func.performReloc(lt_fail_reloc); - } - - const skip_case_reloc = try func.jump(undefined); - - for (liveness.deaths[case.idx]) |operand| try func.processDeath(operand); - - for (relocs) |reloc| func.performReloc(reloc); - try func.genBody(case.body); - try func.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - func.performReloc(skip_case_reloc); - } - - if (switch_br.else_body_len > 0) { - const else_body = it.elseBody(); - - const else_deaths = liveness.deaths.len - 1; - for (liveness.deaths[else_deaths]) |operand| try func.processDeath(operand); - - try func.genBody(else_body); - try func.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - } -} - -fn airLoopSwitchBr(func: *Func, inst: Air.Inst.Index) !void { - const switch_br = func.air.unwrapSwitch(inst); - const condition = try func.resolveInst(switch_br.operand); - - const mat_cond = if (condition.isMutable() and - func.reuseOperand(inst, switch_br.operand, 0, condition)) - condition - else mat_cond: { - const ty = func.typeOf(switch_br.operand); - const mat_cond = try func.allocRegOrMem(ty, inst, true); - try func.genCopy(ty, mat_cond, condition); - break :mat_cond mat_cond; - }; - func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond)); - - // If the condition dies here in this switch instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - if (func.liveness.operandDies(inst, 0)) { - if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); - } - - func.scope_generation += 1; - const state = try func.saveState(); - - try func.loops.putNoClobber(func.gpa, inst, .{ - .state = state, - .jmp_target = @intCast(func.mir_instructions.len), - }); - defer assert(func.loops.remove(inst)); - - // Stop tracking block result without forgetting tracking info - try func.freeValue(mat_cond); - - try func.lowerSwitchBr(inst, switch_br, mat_cond); - - try func.processDeath(inst); - func.finishAirBookkeeping(); -} - -fn airSwitchDispatch(func: *Func, inst: Air.Inst.Index) !void { - const br = func.air.instructions.items(.data)[@intFromEnum(inst)].br; - - const block_ty = func.typeOfIndex(br.block_inst); - const block_tracking = func.inst_tracking.getPtr(br.block_inst).?; - const loop_data = func.loops.getPtr(br.block_inst).?; - done: { - try func.getValue(block_tracking.short, null); - const src_mcv = try func.resolveInst(br.operand); - - if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { - try func.getValue(block_tracking.short, br.block_inst); - // .long = .none to avoid merging operand and block result stack frames. - const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; - try current_tracking.materializeUnsafe(func, br.block_inst, block_tracking.*); - for (current_tracking.getRegs()) |src_reg| func.register_manager.freeReg(src_reg); - break :done; - } - - try func.getValue(block_tracking.short, br.block_inst); - const dst_mcv = block_tracking.short; - try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand)); - break :done; - } - - // Process operand death so that it is properly accounted for in the State below. - if (func.liveness.operandDies(inst, 0)) { - if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); - } - - try func.restoreState(loop_data.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - - // Emit a jump with a relocation. It will be patched up after the block ends. - // Leave the jump offset undefined - _ = try func.jump(loop_data.jmp_target); - - // Stop tracking block result without forgetting tracking info - try func.freeValue(block_tracking.short); - - func.finishAirBookkeeping(); -} - -fn performReloc(func: *Func, inst: Mir.Inst.Index) void { - const tag = func.mir_instructions.items(.tag)[inst]; - const target: Mir.Inst.Index = @intCast(func.mir_instructions.len); - - switch (tag) { - .beq, - .bne, - => func.mir_instructions.items(.data)[inst].b_type.inst = target, - .jal => func.mir_instructions.items(.data)[inst].j_type.inst = target, - .pseudo_j => func.mir_instructions.items(.data)[inst].j_type.inst = target, - else => std.debug.panic("TODO: performReloc {s}", .{@tagName(tag)}), - } -} - -fn airBr(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; - const br = func.air.instructions.items(.data)[@intFromEnum(inst)].br; - - const block_ty = func.typeOfIndex(br.block_inst); - const block_unused = - !block_ty.hasRuntimeBitsIgnoreComptime(zcu) or func.liveness.isUnused(br.block_inst); - const block_tracking = func.inst_tracking.getPtr(br.block_inst).?; - const block_data = func.blocks.getPtr(br.block_inst).?; - const first_br = block_data.relocs.items.len == 0; - const block_result = result: { - if (block_unused) break :result .none; - - if (!first_br) try func.getValue(block_tracking.short, null); - const src_mcv = try func.resolveInst(br.operand); - - if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { - if (first_br) break :result src_mcv; - - try func.getValue(block_tracking.short, br.block_inst); - try InstTracking.materializeUnsafe( - // .long = .none to avoid merging operand and block result stack frames. - .{ .long = .none, .short = src_mcv }, - func, - br.block_inst, - block_tracking.*, - ); - try func.freeValue(src_mcv); - break :result block_tracking.short; - } - - const dst_mcv = if (first_br) try func.allocRegOrMem(block_ty, br.block_inst, true) else dst: { - try func.getValue(block_tracking.short, br.block_inst); - break :dst block_tracking.short; - }; - try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand)); - break :result dst_mcv; - }; - - // Process operand death so that it is properly accounted for in the State below. - if (func.liveness.operandDies(inst, 0)) { - if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); - } - - if (first_br) { - block_tracking.* = InstTracking.init(block_result); - try func.saveRetroactiveState(&block_data.state); - } else try func.restoreState(block_data.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - - // Emit a jump with a relocation. It will be patched up after the block ends. - // Leave the jump offset undefined - const jmp_reloc = try func.jump(undefined); - try block_data.relocs.append(func.gpa, jmp_reloc); - - // Stop tracking block result without forgetting tracking info - try func.freeValue(block_tracking.short); - - func.finishAirBookkeeping(); -} - -fn airRepeat(func: *Func, inst: Air.Inst.Index) !void { - const loop_inst = func.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst; - const repeat_info = func.loops.get(loop_inst).?; - try func.restoreState(repeat_info.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = true, - }); - _ = try func.jump(repeat_info.jmp_target); - func.finishAirBookkeeping(); -} - -fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void { - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const tag: Air.Inst.Tag = func.air.instructions.items(.tag)[@intFromEnum(inst)]; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const lhs = try func.resolveInst(bin_op.lhs); - const rhs = try func.resolveInst(bin_op.rhs); - const lhs_ty = Type.bool; - const rhs_ty = Type.bool; - - const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs); - defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock); - - const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs); - defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock); - - const result_reg, const result_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(result_lock); - - _ = try func.addInst(.{ - .tag = if (tag == .bool_or) .@"or" else .@"and", - .data = .{ .r_type = .{ - .rd = result_reg, - .rs1 = lhs_reg, - .rs2 = rhs_reg, - } }, - }); - - // safety truncate - if (func.wantSafety()) { - _ = try func.addInst(.{ - .tag = .andi, - .data = .{ .i_type = .{ - .rd = result_reg, - .rs1 = result_reg, - .imm12 = Immediate.s(1), - } }, - }); - } - - break :result .{ .register = result_reg }; - }; - return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airAsm(func: *Func, inst: Air.Inst.Index) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Asm, ty_pl.payload); - const outputs_len = extra.data.flags.outputs_len; - var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = - @ptrCast(func.air.extra.items[extra_i..][0..outputs_len]); - extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]); - extra_i += inputs.len; - - var result: MCValue = .none; - var args = std.array_list.Managed(MCValue).init(func.gpa); - try args.ensureTotalCapacity(outputs.len + inputs.len); - defer { - for (args.items) |arg| if (arg.getReg()) |reg| func.register_manager.unlockReg(.{ - .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue, - }); - args.deinit(); - } - var arg_map = std.StringHashMap(u8).init(func.gpa); - try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len)); - defer arg_map.deinit(); - - var outputs_extra_i = extra_i; - for (outputs) |output| { - const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]); - const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0); - const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - const is_read = switch (constraint[0]) { - '=' => false, - '+' => read: { - if (output == .none) return func.fail( - "read-write constraint unsupported for asm result: '{s}'", - .{constraint}, - ); - break :read true; - }, - else => return func.fail("invalid constraint: '{s}'", .{constraint}), - }; - const is_early_clobber = constraint[1] == '&'; - const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..]; - const arg_mcv: MCValue = arg_mcv: { - const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "m")) - if (output != .none) null else return func.fail( - "memory constraint unsupported for asm result: '{s}'", - .{constraint}, - ) - else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}")) - parseRegName(rest["{".len .. rest.len - "}".len]) orelse - return func.fail("invalid register constraint: '{s}'", .{constraint}) - else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { - const index = std.fmt.charToDigit(rest[0], 10) catch unreachable; - if (index >= args.items.len) return func.fail("constraint out of bounds: '{s}'", .{ - constraint, - }); - break :arg_mcv args.items[index]; - } else return func.fail("invalid constraint: '{s}'", .{constraint}); - break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { - const ptr_mcv = try func.resolveInst(output); - switch (ptr_mcv) { - .immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| - break :arg ptr_mcv.deref(), - .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(), - else => {}, - } - break :arg .{ .indirect = .{ .reg = try func.copyToTmpRegister(Type.usize, ptr_mcv) } }; - }; - }; - if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { - _ = func.register_manager.lockReg(reg); - }; - if (!mem.eql(u8, name, "_")) - arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); - args.appendAssumeCapacity(arg_mcv); - if (output == .none) result = arg_mcv; - if (is_read) try func.load(arg_mcv, .{ .air_ref = output }, func.typeOf(output)); - } - - for (inputs) |input| { - const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]); - const constraint = mem.sliceTo(input_bytes, 0); - const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - const ty = func.typeOf(input); - const input_mcv = try func.resolveInst(input); - const arg_mcv: MCValue = if (mem.eql(u8, constraint, "X")) - input_mcv - else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: { - const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse - return func.fail("invalid register constraint: '{s}'", .{constraint}); - try func.register_manager.getReg(reg, null); - try func.genSetReg(ty, reg, input_mcv); - break :arg .{ .register = reg }; - } else if (mem.eql(u8, constraint, "r")) arg: { - switch (input_mcv) { - .register => break :arg input_mcv, - else => {}, - } - const temp_reg = try func.copyToTmpRegister(ty, input_mcv); - break :arg .{ .register = temp_reg }; - } else return func.fail("invalid input constraint: '{s}'", .{constraint}); - if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { - _ = func.register_manager.lockReg(reg); - }; - if (!mem.eql(u8, name, "_")) - arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); - args.appendAssumeCapacity(arg_mcv); - } - - const zcu = func.pt.zcu; - const ip = &zcu.intern_pool; - const aggregate = ip.indexToKey(extra.data.clobbers).aggregate; - const struct_type: Type = .fromInterned(aggregate.ty); - switch (aggregate.storage) { - .elems => |elems| for (elems, 0..) |elem, i| { - switch (elem) { - .bool_true => { - const clobber = struct_type.structFieldName(i, zcu).toSlice(ip).?; - assert(clobber.len != 0); - if (std.mem.eql(u8, clobber, "memory")) { - // nothing really to do - } else { - try func.register_manager.getReg(parseRegName(clobber) orelse - return func.fail("invalid clobber: '{s}'", .{clobber}), null); - } - }, - .bool_false => continue, - else => unreachable, - } - }, - .repeated_elem => |elem| switch (elem) { - .bool_true => @panic("TODO"), - .bool_false => {}, - else => unreachable, - }, - .bytes => @panic("TODO"), - } - - const Label = struct { - target: Mir.Inst.Index = undefined, - pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, - - const Kind = enum { definition, reference }; - - fn isValid(kind: Kind, name: []const u8) bool { - for (name, 0..) |c, i| switch (c) { - else => return false, - '$' => if (i == 0) return false, - '.' => {}, - '0'...'9' => if (i == 0) switch (kind) { - .definition => if (name.len != 1) return false, - .reference => { - if (name.len != 2) return false; - switch (name[1]) { - else => return false, - 'B', 'F', 'b', 'f' => {}, - } - }, - }, - '@', 'A'...'Z', '_', 'a'...'z' => {}, - }; - return name.len > 0; - } - }; - var labels: std.StringHashMapUnmanaged(Label) = .empty; - defer { - var label_it = labels.valueIterator(); - while (label_it.next()) |label| label.pending_relocs.deinit(func.gpa); - labels.deinit(func.gpa); - } - - const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len]; - var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;"); - next_line: while (line_it.next()) |line| { - var mnem_it = mem.tokenizeAny(u8, line, " \t"); - const mnem_str = while (mnem_it.next()) |mnem_str| { - if (mem.startsWith(u8, mnem_str, "#")) continue :next_line; - if (mem.startsWith(u8, mnem_str, "//")) continue :next_line; - if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str; - const label_name = mnem_str[0 .. mnem_str.len - ":".len]; - if (!Label.isValid(.definition, label_name)) - return func.fail("invalid label: '{s}'", .{label_name}); - - const label_gop = try labels.getOrPut(func.gpa, label_name); - if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else { - const anon = std.ascii.isDigit(label_name[0]); - if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0) - return func.fail("redefined label: '{s}'", .{label_name}); - for (label_gop.value_ptr.pending_relocs.items) |pending_reloc| - func.performReloc(pending_reloc); - if (anon) - label_gop.value_ptr.pending_relocs.clearRetainingCapacity() - else - label_gop.value_ptr.pending_relocs.clearAndFree(func.gpa); - } - label_gop.value_ptr.target = @intCast(func.mir_instructions.len); - } else continue; - - const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } = - if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem| - .{ .mnem = mnem } - else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo| - .{ .pseudo = pseudo } - else - return func.fail("invalid mnem str '{s}'", .{mnem_str}); - - const Operand = union(enum) { - none, - reg: Register, - imm: Immediate, - inst: Mir.Inst.Index, - sym: SymbolOffset, - }; - - var ops: [4]Operand = .{.none} ** 4; - var last_op = false; - var op_it = mem.splitAny(u8, mnem_it.rest(), ",("); - next_op: for (&ops) |*op| { - const op_str = while (!last_op) { - const full_str = op_it.next() orelse break :next_op; - const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse - mem.indexOf(u8, full_str, "//")) |comment| - code: { - last_op = true; - break :code full_str[0..comment]; - } else full_str; - const trim_str = mem.trim(u8, code_str, " \t*"); - if (trim_str.len > 0) break trim_str; - } else break; - - if (parseRegName(op_str)) |reg| { - op.* = .{ .reg = reg }; - } else if (std.fmt.parseInt(i12, op_str, 10)) |int| { - op.* = .{ .imm = Immediate.s(int) }; - } else |_| if (mem.startsWith(u8, op_str, "%[")) { - const mod_index = mem.indexOf(u8, op_str, "]@"); - const modifier = if (mod_index) |index| - op_str[index + "]@".len ..] - else - ""; - - op.* = switch (args.items[ - arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse - return func.fail("no matching constraint: '{s}'", .{op_str}) - ]) { - .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { - assert(sym_off.off == 0); - break :blk .{ .sym = sym_off }; - } else return func.fail("invalid modifier: '{s}'", .{modifier}), - .register => |reg| if (modifier.len == 0) - .{ .reg = reg } - else - return func.fail("invalid modified '{s}'", .{modifier}), - else => return func.fail("invalid constraint: '{s}'", .{op_str}), - }; - } else if (mem.endsWith(u8, op_str, ")")) { - const reg = op_str[0 .. op_str.len - ")".len]; - const addr_reg = parseRegName(reg) orelse - return func.fail("expected valid register, found '{s}'", .{reg}); - - op.* = .{ .reg = addr_reg }; - } else if (Label.isValid(.reference, op_str)) { - const anon = std.ascii.isDigit(op_str[0]); - const label_gop = try labels.getOrPut(func.gpa, op_str[0..if (anon) 1 else op_str.len]); - if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; - if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) - return func.fail("undefined label: '{s}'", .{op_str}); - const pending_relocs = &label_gop.value_ptr.pending_relocs; - if (if (anon) - op_str[1] == 'f' or op_str[1] == 'F' - else - !label_gop.found_existing or pending_relocs.items.len > 0) - try pending_relocs.append(func.gpa, @intCast(func.mir_instructions.len)); - op.* = .{ .inst = label_gop.value_ptr.target }; - } else return func.fail("invalid operand: '{s}'", .{op_str}); - } else if (op_it.next()) |op_str| return func.fail("extra operand: '{s}'", .{op_str}); - - switch (instruction) { - .mnem => |mnem| { - _ = (switch (ops[0]) { - .none => try func.addInst(.{ - .tag = mnem, - .data = .none, - }), - .reg => |reg1| switch (ops[1]) { - .reg => |reg2| switch (ops[2]) { - .imm => |imm1| try func.addInst(.{ - .tag = mnem, - .data = .{ .i_type = .{ - .rd = reg1, - .rs1 = reg2, - .imm12 = imm1, - } }, - }), - else => error.InvalidInstruction, - }, - .imm => |imm1| switch (ops[2]) { - .reg => |reg2| switch (mnem) { - .sd => try func.addInst(.{ - .tag = mnem, - .data = .{ .i_type = .{ - .rd = reg2, - .rs1 = reg1, - .imm12 = imm1, - } }, - }), - .ld => try func.addInst(.{ - .tag = mnem, - .data = .{ .i_type = .{ - .rd = reg1, - .rs1 = reg2, - .imm12 = imm1, - } }, - }), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .none => switch (mnem) { - .jalr => try func.addInst(.{ - .tag = mnem, - .data = .{ .i_type = .{ - .rd = .ra, - .rs1 = reg1, - .imm12 = Immediate.s(0), - } }, - }), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }) catch |err| { - switch (err) { - error.InvalidInstruction => return func.fail( - "invalid instruction: {s} {s} {s} {s} {s}", - .{ - @tagName(mnem), - @tagName(ops[0]), - @tagName(ops[1]), - @tagName(ops[2]), - @tagName(ops[3]), - }, - ), - else => |e| return e, - } - }; - }, - .pseudo => |pseudo| { - (@as(error{InvalidInstruction}!void, switch (pseudo) { - .li => blk: { - if (ops[0] != .reg or ops[1] != .imm) { - break :blk error.InvalidInstruction; - } - - const reg = ops[0].reg; - const imm = ops[1].imm; - - try func.genSetReg(Type.usize, reg, .{ .immediate = imm.asBits(u64) }); - }, - .mv => blk: { - if (ops[0] != .reg or ops[1] != .reg) { - break :blk error.InvalidInstruction; - } - - const dst = ops[0].reg; - const src = ops[1].reg; - - if (dst.class() != .int or src.class() != .int) { - return func.fail("pseudo instruction 'mv' only works on integer registers", .{}); - } - - try func.genSetReg(Type.usize, dst, .{ .register = src }); - }, - .tail => blk: { - if (ops[0] != .sym) { - break :blk error.InvalidInstruction; - } - - const sym_offset = ops[0].sym; - assert(sym_offset.off == 0); - - const random_link_reg, const lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(lock); - - _ = try func.addInst(.{ - .tag = .pseudo_extern_fn_reloc, - .data = .{ .reloc = .{ - .register = random_link_reg, - .atom_index = try func.owner.getSymbolIndex(func), - .sym_index = sym_offset.sym, - } }, - }); - }, - .ret => _ = try func.addInst(.{ - .tag = .jalr, - .data = .{ .i_type = .{ - .rd = .zero, - .rs1 = .ra, - .imm12 = Immediate.s(0), - } }, - }), - .beqz => blk: { - if (ops[0] != .reg or ops[1] != .inst) { - break :blk error.InvalidInstruction; - } - - _ = try func.addInst(.{ - .tag = .beq, - .data = .{ .b_type = .{ - .rs1 = ops[0].reg, - .rs2 = .zero, - .inst = ops[1].inst, - } }, - }); - }, - })) catch |err| { - switch (err) { - error.InvalidInstruction => return func.fail( - "invalid instruction: {s} {s} {s} {s} {s}", - .{ - @tagName(pseudo), - @tagName(ops[0]), - @tagName(ops[1]), - @tagName(ops[2]), - @tagName(ops[3]), - }, - ), - else => |e| return e, - } - }; - }, - } - } - - var label_it = labels.iterator(); - while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0) - return func.fail("undefined label: '{s}'", .{label.key_ptr.*}); - - for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { - const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]); - const constraint = - mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0); - const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (output == .none) continue; - if (arg_mcv != .register) continue; - if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue; - try func.store(.{ .air_ref = output }, arg_mcv, func.typeOf(output)); - } - - simple: { - var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); - var buf_index: usize = 0; - for (outputs) |output| { - if (output == .none) continue; - - if (buf_index >= buf.len) break :simple; - buf[buf_index] = output; - buf_index += 1; - } - if (buf_index + inputs.len > buf.len) break :simple; - @memcpy(buf[buf_index..][0..inputs.len], inputs); - return func.finishAir(inst, result, buf); - } - var bt = func.liveness.iterateBigTomb(inst); - for (outputs) |output| if (output != .none) try func.feed(&bt, output); - for (inputs) |input| try func.feed(&bt, input); - return func.finishAirResult(inst, result); -} - -/// Sets the value of `dst_mcv` to the value of `src_mcv`. -fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { - // There isn't anything to store - if (dst_mcv == .none) return; - - if (!dst_mcv.isMutable()) { - // panic so we can see the trace - return std.debug.panic("tried to genCopy immutable: {s}", .{@tagName(dst_mcv)}); - } - - const zcu = func.pt.zcu; - - switch (dst_mcv) { - .register => |reg| return func.genSetReg(ty, reg, src_mcv), - .register_offset => |dst_reg_off| try func.genSetReg(ty, dst_reg_off.reg, switch (src_mcv) { - .none, - .unreach, - .dead, - .undef, - => unreachable, - .immediate, - .register, - .register_offset, - => src_mcv.offset(-dst_reg_off.off), - else => .{ .register_offset = .{ - .reg = try func.copyToTmpRegister(ty, src_mcv), - .off = -dst_reg_off.off, - } }, - }), - .indirect => |reg_off| try func.genSetMem( - .{ .reg = reg_off.reg }, - reg_off.off, - ty, - src_mcv, - ), - .load_frame => |frame_addr| try func.genSetMem( - .{ .frame = frame_addr.index }, - frame_addr.off, - ty, - src_mcv, - ), - .load_symbol => { - const addr_reg, const addr_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(addr_lock); - - try func.genSetReg(ty, addr_reg, dst_mcv.address()); - try func.genCopy(ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); - }, - .memory => return func.fail("TODO: genCopy memory", .{}), - .register_pair => |dst_regs| { - const src_info: ?struct { addr_reg: Register, addr_lock: ?RegisterLock } = switch (src_mcv) { - .register_pair, .memory, .indirect, .load_frame => null, - .load_symbol => src: { - const src_addr_reg, const src_addr_lock = try func.promoteReg(Type.u64, src_mcv.address()); - errdefer func.register_manager.unlockReg(src_addr_lock); - - break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; - }, - .air_ref => |src_ref| return func.genCopy( - ty, - dst_mcv, - try func.resolveInst(src_ref), - ), - else => return func.fail("genCopy register_pair src: {}", .{src_mcv}), - }; - - defer if (src_info) |info| { - if (info.addr_lock) |lock| { - func.register_manager.unlockReg(lock); - } - }; - - var part_disp: i32 = 0; - for (dst_regs, try func.splitType(ty), 0..) |dst_reg, dst_ty, part_i| { - try func.genSetReg(dst_ty, dst_reg, switch (src_mcv) { - .register_pair => |src_regs| .{ .register = src_regs[part_i] }, - .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), - .load_symbol => .{ .indirect = .{ - .reg = src_info.?.addr_reg, - .off = part_disp, - } }, - else => unreachable, - }); - part_disp += @intCast(dst_ty.abiSize(zcu)); - } - }, - else => return std.debug.panic("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }), - } -} - -fn genInlineMemcpy( - func: *Func, - dst_ptr: MCValue, - src_ptr: MCValue, - len: MCValue, -) !void { - const regs = try func.register_manager.allocRegs(4, .{null} ** 4, abi.Registers.Integer.temporary); - const locks = func.register_manager.lockRegsAssumeUnused(4, regs); - defer for (locks) |lock| func.register_manager.unlockReg(lock); - - const count = regs[0]; - const tmp = regs[1]; - const src = regs[2]; - const dst = regs[3]; - - try func.genSetReg(Type.u64, count, len); - try func.genSetReg(Type.u64, src, src_ptr); - try func.genSetReg(Type.u64, dst, dst_ptr); - - // if count is 0, there's nothing to copy - _ = try func.addInst(.{ - .tag = .beq, - .data = .{ .b_type = .{ - .rs1 = count, - .rs2 = .zero, - .inst = @intCast(func.mir_instructions.len + 9), - } }, - }); - - // lb tmp, 0(src) - const first_inst = try func.addInst(.{ - .tag = .lb, - .data = .{ - .i_type = .{ - .rd = tmp, - .rs1 = src, - .imm12 = Immediate.s(0), - }, - }, - }); - - // sb tmp, 0(dst) - _ = try func.addInst(.{ - .tag = .sb, - .data = .{ - .i_type = .{ - .rd = dst, - .rs1 = tmp, - .imm12 = Immediate.s(0), - }, - }, - }); - - // dec count by 1 - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ - .i_type = .{ - .rd = count, - .rs1 = count, - .imm12 = Immediate.s(-1), - }, - }, - }); - - // branch if count is 0 - _ = try func.addInst(.{ - .tag = .beq, - .data = .{ - .b_type = .{ - .inst = @intCast(func.mir_instructions.len + 4), // points after the last inst - .rs1 = count, - .rs2 = .zero, - }, - }, - }); - - // increment the pointers - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ - .i_type = .{ - .rd = src, - .rs1 = src, - .imm12 = Immediate.s(1), - }, - }, - }); - - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ - .i_type = .{ - .rd = dst, - .rs1 = dst, - .imm12 = Immediate.s(1), - }, - }, - }); - - // jump back to start of loop - _ = try func.addInst(.{ - .tag = .pseudo_j, - .data = .{ .j_type = .{ - .rd = .zero, - .inst = first_inst, - } }, - }); -} - -fn genInlineMemset( - func: *Func, - dst_ptr: MCValue, - src_value: MCValue, - len: MCValue, -) !void { - const regs = try func.register_manager.allocRegs(3, .{null} ** 3, abi.Registers.Integer.temporary); - const locks = func.register_manager.lockRegsAssumeUnused(3, regs); - defer for (locks) |lock| func.register_manager.unlockReg(lock); - - const count = regs[0]; - const src = regs[1]; - const dst = regs[2]; - - try func.genSetReg(Type.u64, count, len); - try func.genSetReg(Type.u64, src, src_value); - try func.genSetReg(Type.u64, dst, dst_ptr); - - // sb src, 0(dst) - const first_inst = try func.addInst(.{ - .tag = .sb, - .data = .{ - .i_type = .{ - .rd = dst, - .rs1 = src, - .imm12 = Immediate.s(0), - }, - }, - }); - - // dec count by 1 - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ - .i_type = .{ - .rd = count, - .rs1 = count, - .imm12 = Immediate.s(-1), - }, - }, - }); - - // branch if count is 0 - _ = try func.addInst(.{ - .tag = .beq, - .data = .{ - .b_type = .{ - .inst = @intCast(func.mir_instructions.len + 3), // points after the last inst - .rs1 = count, - .rs2 = .zero, - }, - }, - }); - - // increment the pointers - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ - .i_type = .{ - .rd = dst, - .rs1 = dst, - .imm12 = Immediate.s(1), - }, - }, - }); - - // jump back to start of loop - _ = try func.addInst(.{ - .tag = .pseudo_j, - .data = .{ .j_type = .{ - .rd = .zero, - .inst = first_inst, - } }, - }); -} - -/// Sets the value of `src_mcv` into `reg`. Assumes you have a lock on it. -fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!void { - const pt = func.pt; - const zcu = pt.zcu; - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - - const max_size: u32 = switch (reg.class()) { - .int => 64, - .float => if (func.hasFeature(.d)) 64 else 32, - .vector => 64, // TODO: calculate it from avl * vsew - }; - if (abi_size > max_size) return std.debug.panic("tried to set reg with size {}", .{abi_size}); - const dst_reg_class = reg.class(); - - switch (src_mcv) { - .unreach, - .none, - .dead, - => unreachable, - .undef => |sym_index| { - if (!func.wantSafety()) - return; - - if (sym_index) |index| { - return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } }); - } - - switch (abi_size) { - 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }), - 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }), - 3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }), - 5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }), - else => unreachable, - } - }, - .immediate => |unsigned_x| { - assert(dst_reg_class == .int); - - const x: i64 = @bitCast(unsigned_x); - if (math.minInt(i12) <= x and x <= math.maxInt(i12)) { - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ .i_type = .{ - .rd = reg, - .rs1 = .zero, - .imm12 = Immediate.s(@intCast(x)), - } }, - }); - } else if (math.minInt(i32) <= x and x <= math.maxInt(i32)) { - const lo12: i12 = @truncate(x); - const carry: i32 = if (lo12 < 0) 1 else 0; - const hi20: i20 = @truncate((x >> 12) +% carry); - - _ = try func.addInst(.{ - .tag = .lui, - .data = .{ .u_type = .{ - .rd = reg, - .imm20 = Immediate.s(hi20), - } }, - }); - _ = try func.addInst(.{ - .tag = .addi, - .data = .{ .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.s(lo12), - } }, - }); - } else { - // TODO: use a more advanced myriad seq to do this without a reg. - // see: https://github.com/llvm/llvm-project/blob/081a66ffacfe85a37ff775addafcf3371e967328/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp#L224 - - const temp, const temp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(temp_lock); - - const lo32: i32 = @truncate(x); - const carry: i32 = if (lo32 < 0) 1 else 0; - const hi32: i32 = @truncate((x >> 32) +% carry); - - try func.genSetReg(Type.i32, temp, .{ .immediate = @bitCast(@as(i64, lo32)) }); - try func.genSetReg(Type.i32, reg, .{ .immediate = @bitCast(@as(i64, hi32)) }); - - _ = try func.addInst(.{ - .tag = .slli, - .data = .{ .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(32), - } }, - }); - - _ = try func.addInst(.{ - .tag = .add, - .data = .{ .r_type = .{ - .rd = reg, - .rs1 = reg, - .rs2 = temp, - } }, - }); - } - }, - .register => |src_reg| { - // If the registers are the same, nothing to do. - if (src_reg.id() == reg.id()) - return; - - // there is no instruction for loading the contents of a vector register - // into an integer register, however we can cheat a bit by setting the element - // size to the total size of the vector, and vmv.x.s will work then - if (src_reg.class() == .vector) { - try func.setVl(.zero, 0, .{ - .vsew = switch (ty.totalVectorBits(zcu)) { - 8 => .@"8", - 16 => .@"16", - 32 => .@"32", - 64 => .@"64", - else => |vec_bits| return func.fail("TODO: genSetReg vec -> {s} bits {d}", .{ - @tagName(reg.class()), - vec_bits, - }), - }, - .vlmul = .m1, - .vta = true, - .vma = true, - }); - } - - // mv reg, src_reg - _ = try func.addInst(.{ - .tag = .pseudo_mv, - .data = .{ .rr = .{ - .rd = reg, - .rs = src_reg, - } }, - }); - }, - // useful in cases like slice_ptr, which can easily reuse the operand - // but we need to get only the pointer out. - .register_pair => |pair| try func.genSetReg(ty, reg, .{ .register = pair[0] }), - .load_frame => |frame| { - if (reg.class() == .vector) { - // vectors don't support an offset memory load so we need to put the true - // address into a register before loading from it. - const addr_reg, const addr_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(addr_lock); - - try func.genCopy(ty, .{ .register = addr_reg }, src_mcv.address()); - try func.genCopy(ty, .{ .register = reg }, .{ .indirect = .{ .reg = addr_reg } }); - } else { - _ = try func.addInst(.{ - .tag = .pseudo_load_rm, - .data = .{ .rm = .{ - .r = reg, - .m = .{ - .base = .{ .frame = frame.index }, - .mod = .{ - .size = func.memSize(ty), - .unsigned = ty.isUnsignedInt(zcu), - .disp = frame.off, - }, - }, - } }, - }); - } - }, - .memory => |addr| { - try func.genSetReg(ty, reg, .{ .immediate = addr }); - - _ = try func.addInst(.{ - .tag = .ld, - .data = .{ .i_type = .{ - .rd = reg, - .rs1 = reg, - .imm12 = Immediate.u(0), - } }, - }); - }, - .lea_frame, .register_offset => { - _ = try func.addInst(.{ - .tag = .pseudo_lea_rm, - .data = .{ - .rm = .{ - .r = reg, - .m = switch (src_mcv) { - .register_offset => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ - .size = .byte, // the size doesn't matter - .disp = reg_off.off, - .unsigned = false, - }, - }, - .lea_frame => |frame| .{ - .base = .{ .frame = frame.index }, - .mod = .{ - .size = .byte, // the size doesn't matter - .disp = frame.off, - .unsigned = false, - }, - }, - else => unreachable, - }, - }, - }, - }); - }, - .indirect => |reg_off| { - const load_tag: Mnemonic = switch (reg.class()) { - .float => switch (abi_size) { - 1 => unreachable, // Zig does not support 8-bit floats - 2 => return func.fail("TODO: genSetReg indirect 16-bit float", .{}), - 4 => .flw, - 8 => .fld, - else => return std.debug.panic("TODO: genSetReg for float size {d}", .{abi_size}), - }, - .int => switch (abi_size) { - 1...1 => .lb, - 2...2 => .lh, - 3...4 => .lw, - 5...8 => .ld, - else => return std.debug.panic("TODO: genSetReg for int size {d}", .{abi_size}), - }, - .vector => { - assert(reg_off.off == 0); - - // There is no vector instruction for loading with an offset to a base register, - // so we need to get an offset register containing the address of the vector first - // and load from it. - const len = ty.vectorLen(zcu); - const elem_ty = ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - try func.setVl(.zero, len, .{ - .vsew = switch (elem_size) { - 1 => .@"8", - 2 => .@"16", - 4 => .@"32", - 8 => .@"64", - else => unreachable, - }, - .vlmul = .m1, - .vma = true, - .vta = true, - }); - - _ = try func.addInst(.{ - .tag = .pseudo_load_rm, - .data = .{ .rm = .{ - .r = reg, - .m = .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ - .size = func.memSize(elem_ty), - .unsigned = false, - .disp = 0, - }, - }, - } }, - }); - - return; - }, - }; - - _ = try func.addInst(.{ - .tag = load_tag, - .data = .{ .i_type = .{ - .rd = reg, - .rs1 = reg_off.reg, - .imm12 = Immediate.s(reg_off.off), - } }, - }); - }, - .lea_symbol => |sym_off| { - assert(sym_off.off == 0); - const atom_index = try func.owner.getSymbolIndex(func); - - _ = try func.addInst(.{ - .tag = .pseudo_load_symbol, - .data = .{ .reloc = .{ - .register = reg, - .atom_index = atom_index, - .sym_index = sym_off.sym, - } }, - }); - }, - .load_symbol => { - const addr_reg, const addr_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(addr_lock); - - try func.genSetReg(ty, addr_reg, src_mcv.address()); - try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } }); - }, - .air_ref => |ref| try func.genSetReg(ty, reg, try func.resolveInst(ref)), - else => return func.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}), - } -} - -fn genSetMem( - func: *Func, - base: Memory.Base, - disp: i32, - ty: Type, - src_mcv: MCValue, -) InnerError!void { - const pt = func.pt; - const zcu = pt.zcu; - - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const dst_ptr_mcv: MCValue = switch (base) { - .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, - .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, - }; - switch (src_mcv) { - .none, - .unreach, - .dead, - .reserved_frame, - => unreachable, - .undef => |sym_index| { - if (sym_index) |index| { - return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } }); - } - - try func.genInlineMemset( - dst_ptr_mcv, - src_mcv, - .{ .immediate = abi_size }, - ); - }, - .register_offset, - .memory, - .indirect, - .load_frame, - .lea_frame, - .load_symbol, - .lea_symbol, - => switch (abi_size) { - 0 => {}, - 1, 2, 4, 8 => { - const reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary); - const src_lock = func.register_manager.lockRegAssumeUnused(reg); - defer func.register_manager.unlockReg(src_lock); - - try func.genSetReg(ty, reg, src_mcv); - try func.genSetMem(base, disp, ty, .{ .register = reg }); - }, - else => try func.genInlineMemcpy( - dst_ptr_mcv, - src_mcv.address(), - .{ .immediate = abi_size }, - ), - }, - .register => |reg| { - if (reg.class() == .vector) { - const addr_reg = try func.copyToTmpRegister(Type.u64, dst_ptr_mcv); - - const num_elem = ty.vectorLen(zcu); - const elem_size = ty.childType(zcu).bitSize(zcu); - - try func.setVl(.zero, num_elem, .{ - .vsew = switch (elem_size) { - 8 => .@"8", - 16 => .@"16", - 32 => .@"32", - 64 => .@"64", - else => unreachable, - }, - .vlmul = .m1, - .vma = true, - .vta = true, - }); - - _ = try func.addInst(.{ - .tag = .pseudo_store_rm, - .data = .{ .rm = .{ - .r = reg, - .m = .{ - .base = .{ .reg = addr_reg }, - .mod = .{ - .disp = 0, - .size = func.memSize(ty.childType(zcu)), - .unsigned = false, - }, - }, - } }, - }); - - return; - } - - const mem_size = switch (base) { - .frame => |base_fi| mem_size: { - assert(disp >= 0); - const frame_abi_size = func.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)]; - const frame_spill_pad = func.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)]; - assert(frame_abi_size - frame_spill_pad - disp >= abi_size); - break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size) - frame_abi_size - else - abi_size; - }, - else => abi_size, - }; - const src_size = math.ceilPowerOfTwoAssert(u32, abi_size); - const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size)); - if (src_size > mem_size) { - const frame_index = try func.allocFrameIndex(FrameAlloc.init(.{ - .size = src_size, - .alignment = src_align, - })); - const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } }; - _ = try func.addInst(.{ - .tag = .pseudo_store_rm, - .data = .{ .rm = .{ - .r = reg, - .m = .{ - .base = .{ .frame = frame_index }, - .mod = .{ - .size = Memory.Size.fromByteSize(src_size), - .unsigned = false, - }, - }, - } }, - }); - try func.genSetMem(base, disp, ty, frame_mcv); - try func.freeValue(frame_mcv); - } else _ = try func.addInst(.{ - .tag = .pseudo_store_rm, - .data = .{ .rm = .{ - .r = reg, - .m = .{ - .base = base, - .mod = .{ - .size = func.memSize(ty), - .disp = disp, - .unsigned = false, - }, - }, - } }, - }); - }, - .register_pair => |src_regs| { - var part_disp: i32 = disp; - for (try func.splitType(ty), src_regs) |src_ty, src_reg| { - try func.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }); - part_disp += @intCast(src_ty.abiSize(zcu)); - } - }, - .immediate => { - // TODO: remove this lock in favor of a copyToTmpRegister when we load 64 bit immediates with - // a register allocation. - const reg, const reg_lock = try func.promoteReg(ty, src_mcv); - defer if (reg_lock) |lock| func.register_manager.unlockReg(lock); - - return func.genSetMem(base, disp, ty, .{ .register = reg }); - }, - .air_ref => |src_ref| try func.genSetMem(base, disp, ty, try func.resolveInst(src_ref)), - } -} - -fn airBitCast(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = if (func.liveness.isUnused(inst)) .unreach else result: { - const src_mcv = try func.resolveInst(ty_op.operand); - - const src_ty = func.typeOf(ty_op.operand); - if (src_ty.toIntern() == .bool_type) break :result src_mcv; - const dst_ty = func.typeOfIndex(inst); - - const src_lock = if (src_mcv.getReg()) |reg| func.register_manager.lockReg(reg) else null; - defer if (src_lock) |lock| func.register_manager.unlockReg(lock); - - const dst_mcv = if (dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and src_mcv != .register_pair and - func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { - const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true); - try func.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { - .lt => dst_ty, - .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, - .gt => src_ty, - }, dst_mcv, src_mcv); - break :dst dst_mcv; - }; - - if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and - dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv; - - const abi_size = dst_ty.abiSize(zcu); - const bit_size = dst_ty.bitSize(zcu); - if (abi_size * 8 <= bit_size) break :result dst_mcv; - - return func.fail("TODO: airBitCast {f} to {f}", .{ src_ty.fmt(pt), dst_ty.fmt(pt) }); - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airArrayToSlice(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const slice_ty = func.typeOfIndex(inst); - const ptr_ty = func.typeOf(ty_op.operand); - const ptr = try func.resolveInst(ty_op.operand); - const array_ty = ptr_ty.childType(zcu); - const array_len = array_ty.arrayLen(zcu); - - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu)); - try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr); - try func.genSetMem( - .{ .frame = frame_index }, - @intCast(ptr_ty.abiSize(zcu)), - Type.u64, - .{ .immediate = array_len }, - ); - - const result = MCValue{ .load_frame = .{ .index = frame_index } }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airFloatFromInt(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const pt = func.pt; - const zcu = pt.zcu; - - const operand = try func.resolveInst(ty_op.operand); - - const src_ty = func.typeOf(ty_op.operand); - const dst_ty = ty_op.ty.toType(); - - const src_reg, const src_lock = try func.promoteReg(src_ty, operand); - defer if (src_lock) |lock| func.register_manager.unlockReg(lock); - - const is_unsigned = dst_ty.isUnsignedInt(zcu); - const src_bits = src_ty.bitSize(zcu); - const dst_bits = dst_ty.bitSize(zcu); - - switch (src_bits) { - 32, 64 => {}, - else => try func.truncateRegister(src_ty, src_reg), - } - - const int_zcu: Mir.FcvtOp = switch (src_bits) { - 8, 16, 32 => if (is_unsigned) .wu else .w, - 64 => if (is_unsigned) .lu else .l, - else => return func.fail("TODO: airFloatFromInt src size: {d}", .{src_bits}), - }; - - const float_zcu: enum { s, d } = switch (dst_bits) { - 32 => .s, - 64 => .d, - else => return func.fail("TODO: airFloatFromInt dst size {d}", .{dst_bits}), - }; - - const dst_reg, const dst_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(dst_lock); - - _ = try func.addInst(.{ - .tag = switch (float_zcu) { - .s => switch (int_zcu) { - .l => .fcvtsl, - .lu => .fcvtslu, - .w => .fcvtsw, - .wu => .fcvtswu, - }, - .d => switch (int_zcu) { - .l => .fcvtdl, - .lu => .fcvtdlu, - .w => .fcvtdw, - .wu => .fcvtdwu, - }, - }, - .data = .{ .rr = .{ - .rd = dst_reg, - .rs = src_reg, - } }, - }); - - break :result .{ .register = dst_reg }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIntFromFloat(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const pt = func.pt; - const zcu = pt.zcu; - - const operand = try func.resolveInst(ty_op.operand); - const src_ty = func.typeOf(ty_op.operand); - const dst_ty = ty_op.ty.toType(); - - const is_unsigned = dst_ty.isUnsignedInt(zcu); - const src_bits = src_ty.bitSize(zcu); - const dst_bits = dst_ty.bitSize(zcu); - - const float_zcu: enum { s, d } = switch (src_bits) { - 32 => .s, - 64 => .d, - else => return func.fail("TODO: airIntFromFloat src size {d}", .{src_bits}), - }; - - const int_zcu: Mir.FcvtOp = switch (dst_bits) { - 32 => if (is_unsigned) .wu else .w, - 8, 16, 64 => if (is_unsigned) .lu else .l, - else => return func.fail("TODO: airIntFromFloat dst size: {d}", .{dst_bits}), - }; - - const src_reg, const src_lock = try func.promoteReg(src_ty, operand); - defer if (src_lock) |lock| func.register_manager.unlockReg(lock); - - const dst_reg, const dst_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(dst_lock); - - _ = try func.addInst(.{ - .tag = switch (float_zcu) { - .s => switch (int_zcu) { - .l => .fcvtls, - .lu => .fcvtlus, - .w => .fcvtws, - .wu => .fcvtwus, - }, - .d => switch (int_zcu) { - .l => .fcvtld, - .lu => .fcvtlud, - .w => .fcvtwd, - .wu => .fcvtwud, - }, - }, - .data = .{ .rr = .{ - .rd = dst_reg, - .rs = src_reg, - } }, - }); - - break :result .{ .register = dst_reg }; - }; - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airCmpxchg(func: *Func, inst: Air.Inst.Index, strength: enum { weak, strong }) !void { - _ = strength; // TODO: do something with this - - const pt = func.pt; - const zcu = pt.zcu; - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data; - - const ptr_ty = func.typeOf(extra.ptr); - const val_ty = func.typeOf(extra.expected_value); - const val_abi_size: u32 = @intCast(val_ty.abiSize(pt.zcu)); - - switch (val_abi_size) { - 1, 2, 4, 8 => {}, - else => return func.fail("TODO: airCmpxchg Int size {}", .{val_abi_size}), - } - - const lr_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.successOrder()) { - .unordered, - => unreachable, - - .monotonic, - .release, - => .{ .aq = .none, .rl = .none }, - .acquire, - .acq_rel, - => .{ .aq = .aq, .rl = .none }, - .seq_cst => .{ .aq = .aq, .rl = .rl }, - }; - - const sc_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.failureOrder()) { - .unordered, - .release, - .acq_rel, - => unreachable, - - .monotonic, - .acquire, - .seq_cst, - => switch (extra.successOrder()) { - .release, - .seq_cst, - => .{ .aq = .none, .rl = .rl }, - else => .{ .aq = .none, .rl = .none }, - }, - }; - - const ptr_mcv = try func.resolveInst(extra.ptr); - const ptr_reg, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); - defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); - - const exp_mcv = try func.resolveInst(extra.expected_value); - const exp_reg, const exp_lock = try func.promoteReg(val_ty, exp_mcv); - defer if (exp_lock) |lock| func.register_manager.unlockReg(lock); - try func.truncateRegister(val_ty, exp_reg); - - const new_mcv = try func.resolveInst(extra.new_value); - const new_reg, const new_lock = try func.promoteReg(val_ty, new_mcv); - defer if (new_lock) |lock| func.register_manager.unlockReg(lock); - try func.truncateRegister(val_ty, new_reg); - - const branch_reg, const branch_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(branch_lock); - - const fallthrough_reg, const fallthrough_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(fallthrough_lock); - - const jump_back = try func.addInst(.{ - .tag = if (val_ty.bitSize(zcu) <= 32) .lrw else .lrd, - .data = .{ .amo = .{ - .aq = lr_order.aq, - .rl = lr_order.rl, - .rd = branch_reg, - .rs1 = ptr_reg, - .rs2 = .zero, - } }, - }); - try func.truncateRegister(val_ty, branch_reg); - - const jump_forward = try func.addInst(.{ - .tag = .bne, - .data = .{ .b_type = .{ - .rs1 = branch_reg, - .rs2 = exp_reg, - .inst = undefined, - } }, - }); - - _ = try func.addInst(.{ - .tag = if (val_ty.bitSize(zcu) <= 32) .scw else .scd, - .data = .{ .amo = .{ - .aq = sc_order.aq, - .rl = sc_order.rl, - .rd = fallthrough_reg, - .rs1 = ptr_reg, - .rs2 = new_reg, - } }, - }); - try func.truncateRegister(Type.bool, fallthrough_reg); - - _ = try func.addInst(.{ - .tag = .bne, - .data = .{ .b_type = .{ - .rs1 = fallthrough_reg, - .rs2 = .zero, - .inst = jump_back, - } }, - }); - - func.performReloc(jump_forward); - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, false); - - const tmp_reg, const tmp_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(tmp_lock); - - try func.genBinOp( - .cmp_neq, - .{ .register = branch_reg }, - val_ty, - .{ .register = exp_reg }, - val_ty, - tmp_reg, - ); - - try func.genCopy(val_ty, dst_mcv, .{ .register = branch_reg }); - try func.genCopy( - Type.bool, - dst_mcv.address().offset(@intCast(val_abi_size)).deref(), - .{ .register = tmp_reg }, - ); - - break :result dst_mcv; - }; - - return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); -} - -fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; - - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const op = extra.op(); - const order = extra.ordering(); - - const ptr_ty = func.typeOf(pl_op.operand); - const ptr_mcv = try func.resolveInst(pl_op.operand); - - const val_ty = func.typeOf(extra.operand); - const val_size = val_ty.abiSize(zcu); - const val_mcv = try func.resolveInst(extra.operand); - - if (!math.isPowerOfTwo(val_size)) - return func.fail("TODO: airAtomicRmw non-pow 2", .{}); - - switch (val_ty.zigTypeTag(pt.zcu)) { - .@"enum", .int => {}, - inline .bool, .float, .pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}), - else => unreachable, - } - - const method: enum { amo, loop } = switch (val_size) { - 1, 2 => .loop, - 4, 8 => .amo, - else => unreachable, - }; - - const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); - defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); - - const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv); - defer if (val_lock) |lock| func.register_manager.unlockReg(lock); - - const result_mcv = try func.allocRegOrMem(val_ty, inst, true); - assert(result_mcv == .register); // should fit into 8 bytes - const result_reg = result_mcv.register; - - const aq, const rl = switch (order) { - .unordered => unreachable, - .monotonic => .{ false, false }, - .acquire => .{ true, false }, - .release => .{ false, true }, - .acq_rel => .{ true, true }, - .seq_cst => .{ true, true }, - }; - - switch (method) { - .amo => { - const is_d = val_ty.abiSize(zcu) == 8; - const is_un = val_ty.isUnsignedInt(zcu); - - const mnem: Mnemonic = switch (op) { - // zig fmt: off - .Xchg => if (is_d) .amoswapd else .amoswapw, - .Add => if (is_d) .amoaddd else .amoaddw, - .And => if (is_d) .amoandd else .amoandw, - .Or => if (is_d) .amoord else .amoorw, - .Xor => if (is_d) .amoxord else .amoxorw, - .Max => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw, - .Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw, - else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}), - // zig fmt: on - }; - - _ = try func.addInst(.{ - .tag = mnem, - .data = .{ .amo = .{ - .rd = result_reg, - .rs1 = ptr_register, - .rs2 = val_register, - .aq = if (aq) .aq else .none, - .rl = if (rl) .rl else .none, - } }, - }); - }, - .loop => { - // where we'll jump back when the sc fails - const jump_back = try func.addInst(.{ - .tag = .lrw, - .data = .{ .amo = .{ - .rd = result_reg, - .rs1 = ptr_register, - .rs2 = .zero, - .aq = if (aq) .aq else .none, - .rl = if (rl) .rl else .none, - } }, - }); - - const after_reg, const after_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(after_lock); - - switch (op) { - .Add, .Sub => |tag| { - _ = try func.genBinOp( - switch (tag) { - .Add => .add, - .Sub => .sub, - else => unreachable, - }, - .{ .register = result_reg }, - val_ty, - .{ .register = val_register }, - val_ty, - after_reg, - ); - }, - - else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}), - } - - _ = try func.addInst(.{ - .tag = .scw, - .data = .{ .amo = .{ - .rd = after_reg, - .rs1 = ptr_register, - .rs2 = after_reg, - .aq = if (aq) .aq else .none, - .rl = if (rl) .rl else .none, - } }, - }); - - _ = try func.addInst(.{ - .tag = .bne, - .data = .{ .b_type = .{ - .inst = jump_back, - .rs1 = after_reg, - .rs2 = .zero, - } }, - }); - }, - } - break :result result_mcv; - }; - - return func.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); -} - -fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; - const order: std.builtin.AtomicOrder = atomic_load.order; - - const ptr_ty = func.typeOf(atomic_load.ptr); - const elem_ty = ptr_ty.childType(zcu); - const ptr_mcv = try func.resolveInst(atomic_load.ptr); - - const bit_size = elem_ty.bitSize(zcu); - if (bit_size > 64) return func.fail("TODO: airAtomicLoad > 64 bits", .{}); - - const result_mcv: MCValue = if (func.liveness.isUnused(inst)) - .{ .register = .zero } - else - try func.allocRegOrMem(elem_ty, inst, true); - assert(result_mcv == .register); // should be less than 8 bytes - - if (order == .seq_cst) { - _ = try func.addInst(.{ - .tag = .fence, - .data = .{ .fence = .{ - .pred = .rw, - .succ = .rw, - } }, - }); - } - - try func.load(result_mcv, ptr_mcv, ptr_ty); - - switch (order) { - // Don't guarantee other memory operations to be ordered after the load. - .unordered, .monotonic => {}, - // Make sure all previous reads happen before any reading or writing occurs. - .acquire, .seq_cst => { - _ = try func.addInst(.{ - .tag = .fence, - .data = .{ .fence = .{ - .pred = .r, - .succ = .rw, - } }, - }); - }, - else => unreachable, - } - - return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none }); -} - -fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ptr_ty = func.typeOf(bin_op.lhs); - const ptr_mcv = try func.resolveInst(bin_op.lhs); - - const val_ty = func.typeOf(bin_op.rhs); - const val_mcv = try func.resolveInst(bin_op.rhs); - - const bit_size = val_ty.bitSize(func.pt.zcu); - if (bit_size > 64) return func.fail("TODO: airAtomicStore > 64 bits", .{}); - - switch (order) { - .unordered, .monotonic => {}, - .release, .seq_cst => { - _ = try func.addInst(.{ - .tag = .fence, - .data = .{ .fence = .{ - .pred = .rw, - .succ = .w, - } }, - }); - }, - else => unreachable, - } - - try func.store(ptr_mcv, val_mcv, ptr_ty); - - if (order == .seq_cst) { - _ = try func.addInst(.{ - .tag = .fence, - .data = .{ .fence = .{ - .pred = .rw, - .succ = .rw, - } }, - }); - } - - return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void { - const pt = func.pt; - const zcu = pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - result: { - if (!safety and (try func.resolveInst(bin_op.rhs)) == .undef) break :result; - - const dst_ptr = try func.resolveInst(bin_op.lhs); - const dst_ptr_ty = func.typeOf(bin_op.lhs); - const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (dst_ptr_lock) |lock| func.register_manager.unlockReg(lock); - - const src_val = try func.resolveInst(bin_op.rhs); - const elem_ty = func.typeOf(bin_op.rhs); - const src_val_lock: ?RegisterLock = switch (src_val) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (src_val_lock) |lock| func.register_manager.unlockReg(lock); - - const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu)); - - if (elem_abi_size == 1) { - const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { - // TODO: this only handles slices stored in the stack - .slice => dst_ptr, - .one => dst_ptr, - .c, .many => unreachable, - }; - const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { - // TODO: this only handles slices stored in the stack - .slice => dst_ptr.address().offset(8).deref(), - .one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) }, - .c, .many => unreachable, - }; - const len_lock: ?RegisterLock = switch (len) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (len_lock) |lock| func.register_manager.unlockReg(lock); - - try func.genInlineMemset(ptr, src_val, len); - break :result; - } - - // Store the first element, and then rely on memcpy copying forwards. - // Length zero requires a runtime check - so we handle arrays specially - // here to elide it. - switch (dst_ptr_ty.ptrSize(zcu)) { - .slice => return func.fail("TODO: airMemset Slices", .{}), - .one => { - const elem_ptr_ty = try pt.singleMutPtrType(elem_ty); - - const len = dst_ptr_ty.childType(zcu).arrayLen(zcu); - - assert(len != 0); // prevented by Sema - try func.store(dst_ptr, src_val, elem_ptr_ty); - - const second_elem_ptr_reg, const second_elem_ptr_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(second_elem_ptr_lock); - - const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; - - try func.genSetReg(Type.u64, second_elem_ptr_reg, .{ .register_offset = .{ - .reg = try func.copyToTmpRegister(Type.u64, dst_ptr), - .off = elem_abi_size, - } }); - - const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; - try func.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); - }, - .c, .many => unreachable, - } - } - return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMemcpy(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const dst_ptr = try func.resolveInst(bin_op.lhs); - const src_ptr = try func.resolveInst(bin_op.rhs); - - const dst_ty = func.typeOf(bin_op.lhs); - - const len_mcv: MCValue = switch (dst_ty.ptrSize(zcu)) { - .slice => len: { - const len_reg, const len_lock = try func.allocReg(.int); - defer func.register_manager.unlockReg(len_lock); - - const elem_size = dst_ty.childType(zcu).abiSize(zcu); - try func.genBinOp( - .mul, - .{ .immediate = elem_size }, - Type.u64, - dst_ptr.address().offset(8).deref(), - Type.u64, - len_reg, - ); - break :len .{ .register = len_reg }; - }, - .one => len: { - const array_ty = dst_ty.childType(zcu); - break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) }; - }, - else => |size| return func.fail("TODO: airMemcpy size {s}", .{@tagName(size)}), - }; - const len_lock: ?RegisterLock = switch (len_mcv) { - .register => |reg| func.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (len_lock) |lock| func.register_manager.unlockReg(lock); - - try func.genInlineMemcpy(dst_ptr, src_ptr, len_mcv); - - return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMemmove(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement airMemmove for riscv64", .{}); -} - -fn airTagName(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - - const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { - const enum_ty = func.typeOf(un_op); - - // TODO: work out the bugs - if (true) return func.fail("TODO: airTagName", .{}); - - const param_regs = abi.Registers.Integer.function_arg_regs; - const dst_mcv = try func.allocRegOrMem(Type.u64, inst, false); - try func.genSetReg(Type.u64, param_regs[0], dst_mcv.address()); - - const operand = try func.resolveInst(un_op); - try func.genSetReg(enum_ty, param_regs[1], operand); - - const lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() }; - const elf_file = func.bin_file.cast(link.File.Elf).?; - const zo = elf_file.zigObjectPtr().?; - const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err| - return func.fail("{s} creating lazy symbol", .{@errorName(err)}); - - if (func.mod.pic) { - return func.fail("TODO: airTagName pic", .{}); - } else { - try func.genSetReg(Type.u64, .ra, .{ .load_symbol = .{ .sym = sym_index } }); - _ = try func.addInst(.{ - .tag = .jalr, - .data = .{ .i_type = .{ - .rd = .ra, - .rs1 = .ra, - .imm12 = Immediate.s(0), - } }, - }); - } - - break :result dst_mcv; - }; - return func.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airErrorName(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO: airErrorName", .{}); -} - -fn airSplat(func: *Func, inst: Air.Inst.Index) !void { - const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSplat for riscv64", .{}); - return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSelect(func: *Func, inst: Air.Inst.Index) !void { - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = func.air.extraData(Air.Bin, pl_op.payload).data; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSelect for riscv64", .{}); - return func.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); -} - -fn airShuffleOne(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement airShuffleOne for riscv64", .{}); -} - -fn airShuffleTwo(func: *Func, inst: Air.Inst.Index) !void { - _ = inst; - return func.fail("TODO implement airShuffleTwo for riscv64", .{}); -} - -fn airReduce(func: *Func, inst: Air.Inst.Index) !void { - const reduce = func.air.instructions.items(.data)[@intFromEnum(inst)].reduce; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airReduce for riscv64", .{}); - return func.finishAir(inst, result, .{ reduce.operand, .none, .none }); -} - -fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void { - const pt = func.pt; - const zcu = pt.zcu; - const result_ty = func.typeOfIndex(inst); - const len: usize = @intCast(result_ty.arrayLen(zcu)); - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]); - - const result: MCValue = result: { - switch (result_ty.zigTypeTag(zcu)) { - .@"struct" => { - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu)); - if (result_ty.containerLayout(zcu) == .@"packed") { - const struct_obj = zcu.typeToStruct(result_ty).?; - try func.genInlineMemset( - .{ .lea_frame = .{ .index = frame_index } }, - .{ .immediate = 0 }, - .{ .immediate = result_ty.abiSize(zcu) }, - ); - - for (elements, 0..) |elem, elem_i_usize| { - const elem_i: u32 = @intCast(elem_i_usize); - if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; - - const elem_ty = result_ty.fieldType(elem_i, zcu); - const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu)); - if (elem_bit_size > 64) { - return func.fail( - "TODO airAggregateInit implement packed structs with large fields", - .{}, - ); - } - - const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); - const elem_abi_bits = elem_abi_size * 8; - const elem_off = zcu.structPackedFieldBitOffset(struct_obj, elem_i); - const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size); - const elem_bit_off = elem_off % elem_abi_bits; - const elem_mcv = try func.resolveInst(elem); - - _ = elem_byte_off; - _ = elem_bit_off; - - const elem_lock = switch (elem_mcv) { - .register => |reg| func.register_manager.lockReg(reg), - .immediate => |imm| lock: { - if (imm == 0) continue; - break :lock null; - }, - else => null, - }; - defer if (elem_lock) |lock| func.register_manager.unlockReg(lock); - - return func.fail("TODO: airAggregateInit packed structs", .{}); - } - } else for (elements, 0..) |elem, elem_i| { - if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; - - const elem_ty = result_ty.fieldType(elem_i, zcu); - const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu)); - const elem_mcv = try func.resolveInst(elem); - try func.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv); - } - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - .array => { - const elem_ty = result_ty.childType(zcu); - const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu)); - const elem_size: u32 = @intCast(elem_ty.abiSize(zcu)); - - for (elements, 0..) |elem, elem_i| { - const elem_mcv = try func.resolveInst(elem); - const elem_off: i32 = @intCast(elem_size * elem_i); - try func.genSetMem( - .{ .frame = frame_index }, - elem_off, - elem_ty, - elem_mcv, - ); - } - if (result_ty.sentinel(zcu)) |sentinel| try func.genSetMem( - .{ .frame = frame_index }, - @intCast(elem_size * elements.len), - elem_ty, - try func.genTypedValue(sentinel), - ); - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - else => return func.fail("TODO: airAggregate {f}", .{result_ty.fmt(pt)}), - } - }; - - if (elements.len <= Air.Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); - @memcpy(buf[0..elements.len], elements); - return func.finishAir(inst, result, buf); - } - var bt = func.liveness.iterateBigTomb(inst); - for (elements) |elem| try func.feed(&bt, elem); - return func.finishAirResult(inst, result); -} - -fn airUnionInit(func: *Func, inst: Air.Inst.Index) !void { - const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = func.air.extraData(Air.UnionInit, ty_pl.payload).data; - _ = extra; - return func.fail("TODO implement airUnionInit for riscv64", .{}); - // return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); -} - -fn airPrefetch(func: *Func, inst: Air.Inst.Index) !void { - const prefetch = func.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; - // TODO: RISC-V does have prefetch instruction variants. - // see here: https://raw.githubusercontent.com/riscv/riscv-CMOs/master/specifications/cmobase-v1.0.1.pdf - return func.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none }); -} - -fn airMulAdd(func: *Func, inst: Air.Inst.Index) !void { - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = func.air.extraData(Air.Bin, pl_op.payload).data; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else { - return func.fail("TODO implement airMulAdd for riscv64", .{}); - }; - return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand }); -} - -fn resolveInst(func: *Func, ref: Air.Inst.Ref) InnerError!MCValue { - const pt = func.pt; - const zcu = pt.zcu; - - // If the type has no codegen bits, no need to store it. - const inst_ty = func.typeOf(ref); - if (!inst_ty.hasRuntimeBits(zcu)) - return .none; - - const mcv = if (ref.toIndex()) |inst| mcv: { - break :mcv func.inst_tracking.getPtr(inst).?.short; - } else mcv: { - const ip_index = ref.toInterned().?; - const gop = try func.const_tracking.getOrPut(func.gpa, ip_index); - if (!gop.found_existing) gop.value_ptr.* = InstTracking.init( - try func.genTypedValue(Value.fromInterned(ip_index)), - ); - break :mcv gop.value_ptr.short; - }; - - return mcv; -} - -fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking { - const tracking = func.inst_tracking.getPtr(inst).?; - return switch (tracking.short) { - .none, .unreach, .dead => unreachable, - else => tracking, - }; -} - -fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { - const pt = func.pt; - - const lf = func.bin_file; - const src_loc = func.src_loc; - - const result: codegen.GenResult = if (val.isUndef(pt.zcu)) - switch (try lf.lowerUav(pt, val.toIntern(), .none, src_loc)) { - .sym_index => |sym_index| .{ .mcv = .{ .load_symbol = sym_index } }, - .fail => |em| .{ .fail = em }, - } - else - try codegen.genTypedValue(lf, pt, src_loc, val, func.target); - const mcv: MCValue = switch (result) { - .mcv => |mcv| switch (mcv) { - .none => .none, - .undef => unreachable, - .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } }, - .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, - .immediate => |imm| .{ .immediate = imm }, - .memory => |addr| .{ .memory = addr }, - .load_got, .load_direct, .lea_direct => { - return func.fail("TODO: genTypedValue {s}", .{@tagName(mcv)}); - }, - }, - .fail => |msg| return func.failMsg(msg), - }; - return mcv; -} - -const CallMCValues = struct { - args: []MCValue, - return_value: InstTracking, - stack_byte_count: u31, - stack_align: Alignment, - - fn deinit(call: *CallMCValues, func: *Func) void { - func.gpa.free(call.args); - call.* = undefined; - } -}; - -/// Caller must call `CallMCValues.deinit`. -fn resolveCallingConventionValues( - func: *Func, - fn_info: InternPool.Key.FuncType, - var_args: []const Type, -) !CallMCValues { - const pt = func.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const param_types = try func.gpa.alloc(Type, fn_info.param_types.len + var_args.len); - defer func.gpa.free(param_types); - - for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| { - dest.* = Type.fromInterned(src); - } - for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty| - param_ty.* = func.promoteVarArg(arg_ty); - - const cc = fn_info.cc; - var result: CallMCValues = .{ - .args = try func.gpa.alloc(MCValue, param_types.len), - // These undefined values must be populated before returning from this function. - .return_value = undefined, - .stack_byte_count = 0, - .stack_align = undefined, - }; - errdefer func.gpa.free(result.args); - - const ret_ty = Type.fromInterned(fn_info.return_type); - - switch (cc) { - .naked => { - assert(result.args.len == 0); - result.return_value = InstTracking.init(.unreach); - result.stack_align = .@"8"; - }, - .riscv64_lp64, .auto => { - if (result.args.len > 8) { - return func.fail("RISC-V calling convention does not support more than 8 arguments", .{}); - } - - var ret_int_reg_i: u32 = 0; - var param_int_reg_i: u32 = 0; - - result.stack_align = .@"16"; - - // Return values - if (ret_ty.zigTypeTag(zcu) == .noreturn) { - result.return_value = InstTracking.init(.unreach); - } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - result.return_value = InstTracking.init(.none); - } else { - var ret_tracking: [2]InstTracking = undefined; - var ret_tracking_i: usize = 0; - var ret_float_reg_i: usize = 0; - - const classes = mem.sliceTo(&abi.classifySystem(ret_ty, zcu), .none); - - for (classes) |class| switch (class) { - .integer => { - const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i]; - ret_int_reg_i += 1; - - ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg }); - ret_tracking_i += 1; - }, - .float => { - const ret_float_reg = abi.Registers.Float.function_ret_regs[ret_float_reg_i]; - ret_float_reg_i += 1; - - ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_float_reg }); - ret_tracking_i += 1; - }, - .memory => { - const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i]; - ret_int_reg_i += 1; - const ret_indirect_reg = abi.Registers.Integer.function_arg_regs[param_int_reg_i]; - param_int_reg_i += 1; - - ret_tracking[ret_tracking_i] = .{ - .short = .{ .indirect = .{ .reg = ret_int_reg } }, - .long = .{ .indirect = .{ .reg = ret_indirect_reg } }, - }; - ret_tracking_i += 1; - }, - else => return func.fail("TODO: C calling convention return class {}", .{class}), - }; - - result.return_value = switch (ret_tracking_i) { - else => return func.fail("ty {f} took {} tracking return indices", .{ ret_ty.fmt(pt), ret_tracking_i }), - 1 => ret_tracking[0], - 2 => InstTracking.init(.{ .register_pair = .{ - ret_tracking[0].short.register, ret_tracking[1].short.register, - } }), - }; - } - - var param_float_reg_i: usize = 0; - - for (param_types, result.args) |ty, *arg| { - if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) { - assert(cc == .auto); - arg.* = .none; - continue; - } - - var arg_mcv: [2]MCValue = undefined; - var arg_mcv_i: usize = 0; - - const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none); - - for (classes) |class| switch (class) { - .integer => { - const param_int_regs = abi.Registers.Integer.function_arg_regs; - if (param_int_reg_i >= param_int_regs.len) break; - - const param_int_reg = param_int_regs[param_int_reg_i]; - param_int_reg_i += 1; - - arg_mcv[arg_mcv_i] = .{ .register = param_int_reg }; - arg_mcv_i += 1; - }, - .float => { - const param_float_regs = abi.Registers.Float.function_arg_regs; - if (param_float_reg_i >= param_float_regs.len) break; - - const param_float_reg = param_float_regs[param_float_reg_i]; - param_float_reg_i += 1; - - arg_mcv[arg_mcv_i] = .{ .register = param_float_reg }; - arg_mcv_i += 1; - }, - .memory => { - const param_int_regs = abi.Registers.Integer.function_arg_regs; - - const param_int_reg = param_int_regs[param_int_reg_i]; - param_int_reg_i += 1; - - arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } }; - arg_mcv_i += 1; - }, - else => return func.fail("TODO: C calling convention arg class {}", .{class}), - } else { - arg.* = switch (arg_mcv_i) { - else => return func.fail("ty {f} took {} tracking arg indices", .{ ty.fmt(pt), arg_mcv_i }), - 1 => arg_mcv[0], - 2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } }, - }; - continue; - } - - return func.fail("TODO: pass args by stack", .{}); - } - }, - else => return func.fail("TODO implement function parameters for {} on riscv64", .{cc}), - } - - result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count)); - return result; -} - -fn wantSafety(func: *Func) bool { - return switch (func.mod.optimize_mode) { - .Debug => true, - .ReleaseSafe => true, - .ReleaseFast => false, - .ReleaseSmall => false, - }; -} - -fn fail(func: *const Func, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = func.pt.zcu; - switch (func.owner) { - .nav_index => |i| return zcu.codegenFail(i, format, args), - .lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args), - } - return error.CodegenFail; -} - -fn failMsg(func: *const Func, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = func.pt.zcu; - switch (func.owner) { - .nav_index => |i| return zcu.codegenFailMsg(i, msg), - .lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg), - } - return error.CodegenFail; -} - -fn parseRegName(name: []const u8) ?Register { - // The `fp` alias for `s0` is awkward to fit into the current `Register` scheme, so for now we - // special-case it here. - if (std.mem.eql(u8, name, "fp")) return .s0; - - return std.meta.stringToEnum(Register, name); -} - -fn typeOf(func: *Func, inst: Air.Inst.Ref) Type { - return func.air.typeOf(inst, &func.pt.zcu.intern_pool); -} - -fn typeOfIndex(func: *Func, inst: Air.Inst.Index) Type { - const zcu = func.pt.zcu; - return switch (func.air.instructions.items(.tag)[@intFromEnum(inst)]) { - .loop_switch_br => func.typeOf(func.air.unwrapSwitch(inst).operand), - else => func.air.typeOfIndex(inst, &zcu.intern_pool), - }; -} - -fn hasFeature(func: *Func, feature: Target.riscv.Feature) bool { - return func.target.cpu.has(.riscv, feature); -} - -pub fn errUnionPayloadOffset(payload_ty: Type, zcu: *Zcu) u64 { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0; - const payload_align = payload_ty.abiAlignment(zcu); - const error_align = Type.anyerror.abiAlignment(zcu); - if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return 0; - } else { - return payload_align.forward(Type.anyerror.abiSize(zcu)); - } -} - -pub fn errUnionErrorOffset(payload_ty: Type, zcu: *Zcu) u64 { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0; - const payload_align = payload_ty.abiAlignment(zcu); - const error_align = Type.anyerror.abiAlignment(zcu); - if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return error_align.forward(payload_ty.abiSize(zcu)); - } else { - return 0; - } -} - -fn promoteInt(func: *Func, ty: Type) Type { - const pt = func.pt; - const zcu = pt.zcu; - const int_info: InternPool.Key.IntType = switch (ty.toIntern()) { - .bool_type => .{ .signedness = .unsigned, .bits = 1 }, - else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty, - }; - for ([_]Type{ - Type.c_int, Type.c_uint, - Type.c_long, Type.c_ulong, - Type.c_longlong, Type.c_ulonglong, - }) |promote_ty| { - const promote_info = promote_ty.intInfo(zcu); - if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue; - if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and - promote_info.signedness == .signed) <= promote_info.bits) return promote_ty; - } - return ty; -} - -fn promoteVarArg(func: *Func, ty: Type) Type { - if (!ty.isRuntimeFloat()) return func.promoteInt(ty); - switch (ty.floatBits(func.target)) { - 32, 64 => return Type.f64, - else => |float_bits| { - assert(float_bits == func.target.cTypeBitSize(.longdouble)); - return Type.c_longdouble; - }, - } -} diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig deleted file mode 100644 index 64a476007cf9881e5101c5c945158252b1ba5b86..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/Emit.zig +++ /dev/null @@ -1,222 +0,0 @@ -//! This file contains the functionality for emitting RISC-V MIR as machine code - -bin_file: *link.File, -lower: Lower, -debug_output: link.File.DebugInfoOutput, -w: *std.Io.Writer, - -prev_di_line: u32, -prev_di_column: u32, -/// Relative to the beginning of `code`. -prev_di_pc: usize, - -code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty, -relocs: std.ArrayListUnmanaged(Reloc) = .empty, - -pub const Error = Lower.Error || std.Io.Writer.Error || error{ - EmitFail, -}; - -pub fn emitMir(emit: *Emit) Error!void { - const gpa = emit.bin_file.comp.gpa; - log.debug("mir instruction len: {}", .{emit.lower.mir.instructions.len}); - for (0..emit.lower.mir.instructions.len) |mir_i| { - const mir_index: Mir.Inst.Index = @intCast(mir_i); - try emit.code_offset_mapping.putNoClobber( - emit.lower.allocator, - mir_index, - @intCast(emit.w.end), - ); - const lowered = try emit.lower.lowerMir(mir_index, .{ .allow_frame_locs = true }); - var lowered_relocs = lowered.relocs; - for (lowered.insts, 0..) |lowered_inst, lowered_index| { - const start_offset: u32 = @intCast(emit.w.end); - try emit.w.writeInt(u32, lowered_inst.toU32(), .little); - - while (lowered_relocs.len > 0 and - lowered_relocs[0].lowered_inst_index == lowered_index) : ({ - lowered_relocs = lowered_relocs[1..]; - }) switch (lowered_relocs[0].target) { - .inst => |target| try emit.relocs.append(emit.lower.allocator, .{ - .source = start_offset, - .target = target, - .offset = 0, - .fmt = std.meta.activeTag(lowered_inst), - }), - .load_symbol_reloc => |symbol| { - const elf_file = emit.bin_file.cast(.elf).?; - const zo = elf_file.zigObjectPtr().?; - - const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; - const sym = zo.symbol(symbol.sym_index); - - if (emit.lower.pic) { - return emit.fail("know when to emit GOT relocation for symbol '{s}'", .{sym.name(elf_file)}); - } - - const hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); - const lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); - - try atom_ptr.addReloc(gpa, .{ - .r_offset = start_offset, - .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | hi_r_type, - .r_addend = 0, - }, zo); - - try atom_ptr.addReloc(gpa, .{ - .r_offset = start_offset + 4, - .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | lo_r_type, - .r_addend = 0, - }, zo); - }, - .load_tlv_reloc => |symbol| { - const elf_file = emit.bin_file.cast(.elf).?; - const zo = elf_file.zigObjectPtr().?; - - const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; - - const R_RISCV = std.elf.R_RISCV; - - try atom_ptr.addReloc(gpa, .{ - .r_offset = start_offset, - .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | @intFromEnum(R_RISCV.TPREL_HI20), - .r_addend = 0, - }, zo); - - try atom_ptr.addReloc(gpa, .{ - .r_offset = start_offset + 4, - .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | @intFromEnum(R_RISCV.TPREL_ADD), - .r_addend = 0, - }, zo); - - try atom_ptr.addReloc(gpa, .{ - .r_offset = start_offset + 8, - .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | @intFromEnum(R_RISCV.TPREL_LO12_I), - .r_addend = 0, - }, zo); - }, - .call_extern_fn_reloc => |symbol| { - const elf_file = emit.bin_file.cast(.elf).?; - const zo = elf_file.zigObjectPtr().?; - const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; - - const r_type: u32 = @intFromEnum(std.elf.R_RISCV.CALL_PLT); - - try atom_ptr.addReloc(gpa, .{ - .r_offset = start_offset, - .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | r_type, - .r_addend = 0, - }, zo); - }, - }; - } - std.debug.assert(lowered_relocs.len == 0); - - if (lowered.insts.len == 0) { - const mir_inst = emit.lower.mir.instructions.get(mir_index); - switch (mir_inst.tag) { - else => unreachable, - .pseudo_dbg_prologue_end => { - switch (emit.debug_output) { - .dwarf => |dw| { - try dw.setPrologueEnd(); - log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{ - emit.prev_di_line, emit.prev_di_column, - }); - try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); - }, - .none => {}, - } - }, - .pseudo_dbg_line_column => try emit.dbgAdvancePCAndLine( - mir_inst.data.pseudo_dbg_line_column.line, - mir_inst.data.pseudo_dbg_line_column.column, - ), - .pseudo_dbg_epilogue_begin => { - switch (emit.debug_output) { - .dwarf => |dw| { - try dw.setEpilogueBegin(); - log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{ - emit.prev_di_line, emit.prev_di_column, - }); - try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); - }, - .none => {}, - } - }, - .pseudo_dead => {}, - } - } - } - try emit.fixupRelocs(); -} - -pub fn deinit(emit: *Emit) void { - emit.relocs.deinit(emit.lower.allocator); - emit.code_offset_mapping.deinit(emit.lower.allocator); - emit.* = undefined; -} - -const Reloc = struct { - /// Offset of the instruction. - source: usize, - /// Target of the relocation. - target: Mir.Inst.Index, - /// Offset of the relocation within the instruction. - offset: u32, - /// Format of the instruction, used to determine how to modify it. - fmt: encoding.Lir.Format, -}; - -fn fixupRelocs(emit: *Emit) Error!void { - for (emit.relocs.items) |reloc| { - log.debug("target inst: {f}", .{emit.lower.mir.instructions.get(reloc.target)}); - const target = emit.code_offset_mapping.get(reloc.target) orelse - return emit.fail("relocation target not found!", .{}); - - const disp = @as(i32, @intCast(target)) - @as(i32, @intCast(reloc.source)); - const code = emit.w.buffered()[reloc.source + reloc.offset ..][0..4]; - - switch (reloc.fmt) { - .J => riscv_util.writeInstJ(code, @bitCast(disp)), - .B => riscv_util.writeInstB(code, @bitCast(disp)), - else => return emit.fail("tried to reloc format type {s}", .{@tagName(reloc.fmt)}), - } - } -} - -fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) Error!void { - const delta_line = @as(i33, line) - @as(i33, emit.prev_di_line); - const delta_pc: usize = emit.w.end - emit.prev_di_pc; - log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line }); - switch (emit.debug_output) { - .dwarf => |dw| { - if (column != emit.prev_di_column) try dw.setColumn(column); - if (delta_line == 0) return; // TODO: fix these edge cases. - try dw.advancePCAndLine(delta_line, delta_pc); - emit.prev_di_line = line; - emit.prev_di_column = column; - emit.prev_di_pc = emit.w.end; - }, - .none => {}, - } -} - -fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error { - return switch (emit.lower.fail(format, args)) { - error.LowerFail => error.EmitFail, - else => |e| e, - }; -} - -const link = @import("../../link.zig"); -const log = std.log.scoped(.emit); -const mem = std.mem; -const std = @import("std"); - -const Emit = @This(); -const Lower = @import("Lower.zig"); -const Mir = @import("Mir.zig"); -const riscv_util = @import("../../link/riscv.zig"); -const Elf = @import("../../link/Elf.zig"); -const encoding = @import("encoding.zig"); diff --git a/src/arch/riscv64/Lower.zig b/src/arch/riscv64/Lower.zig deleted file mode 100644 index 08bb3e2b937b448492030a76d5467dec24da9e21..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/Lower.zig +++ /dev/null @@ -1,615 +0,0 @@ -//! This file contains the functionality for lowering RISC-V MIR to Instructions - -pt: Zcu.PerThread, -output_mode: std.builtin.OutputMode, -link_mode: std.builtin.LinkMode, -pic: bool, -allocator: Allocator, -mir: Mir, -cc: std.builtin.CallingConvention, -err_msg: ?*ErrorMsg = null, -src_loc: Zcu.LazySrcLoc, -result_insts_len: u8 = undefined, -result_relocs_len: u8 = undefined, -result_insts: [ - @max( - 1, // non-pseudo instruction - abi.Registers.all_preserved.len, // spill / restore regs, - ) -]Instruction = undefined, -result_relocs: [1]Reloc = undefined, - -pub const Error = error{ - OutOfMemory, - LowerFail, - InvalidInstruction, -}; - -pub const Reloc = struct { - lowered_inst_index: u8, - target: Target, - - const Target = union(enum) { - inst: Mir.Inst.Index, - - /// Relocs the lowered_inst_index and the next instruction. - load_symbol_reloc: bits.Symbol, - /// Relocs the lowered_inst_index and the next two instructions. - load_tlv_reloc: bits.Symbol, - /// Relocs the lowered_inst_index and the next instruction. - call_extern_fn_reloc: bits.Symbol, - }; -}; - -/// The returned slice is overwritten by the next call to lowerMir. -pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct { - allow_frame_locs: bool, -}) Error!struct { - insts: []const Instruction, - relocs: []const Reloc, -} { - const pt = lower.pt; - const zcu = pt.zcu; - - lower.result_insts = undefined; - lower.result_relocs = undefined; - errdefer lower.result_insts = undefined; - errdefer lower.result_relocs = undefined; - lower.result_insts_len = 0; - lower.result_relocs_len = 0; - defer lower.result_insts_len = undefined; - defer lower.result_relocs_len = undefined; - - const inst = lower.mir.instructions.get(index); - log.debug("lowerMir {f}", .{inst}); - switch (inst.tag) { - else => try lower.generic(inst), - .pseudo_dbg_line_column, - .pseudo_dbg_epilogue_begin, - .pseudo_dbg_prologue_end, - .pseudo_dead, - => {}, - - .pseudo_load_rm, .pseudo_store_rm => { - const rm = inst.data.rm; - - const frame_loc: Mir.FrameLoc = if (options.allow_frame_locs) - rm.m.toFrameLoc(lower.mir) - else - .{ .base = .s0, .disp = 0 }; - - switch (inst.tag) { - .pseudo_load_rm => { - const dest_reg = rm.r; - const dest_reg_class = dest_reg.class(); - - const src_size = rm.m.mod.size; - const unsigned = rm.m.mod.unsigned; - - const mnem: Mnemonic = switch (dest_reg_class) { - .int => switch (src_size) { - .byte => if (unsigned) .lbu else .lb, - .hword => if (unsigned) .lhu else .lh, - .word => if (unsigned) .lwu else .lw, - .dword => .ld, - }, - .float => switch (src_size) { - .byte => unreachable, // Zig does not support 8-bit floats - .hword => return lower.fail("TODO: lowerMir pseudo_load_rm support 16-bit floats", .{}), - .word => .flw, - .dword => .fld, - }, - .vector => switch (src_size) { - .byte => .vle8v, - .hword => .vle32v, - .word => .vle32v, - .dword => .vle64v, - }, - }; - - switch (dest_reg_class) { - .int, .float => { - try lower.emit(mnem, &.{ - .{ .reg = rm.r }, - .{ .reg = frame_loc.base }, - .{ .imm = Immediate.s(frame_loc.disp) }, - }); - }, - .vector => { - assert(frame_loc.disp == 0); - try lower.emit(mnem, &.{ - .{ .reg = rm.r }, - .{ .reg = frame_loc.base }, - .{ .reg = .zero }, - }); - }, - } - }, - .pseudo_store_rm => { - const src_reg = rm.r; - const src_reg_class = src_reg.class(); - - const dest_size = rm.m.mod.size; - - const mnem: Mnemonic = switch (src_reg_class) { - .int => switch (dest_size) { - .byte => .sb, - .hword => .sh, - .word => .sw, - .dword => .sd, - }, - .float => switch (dest_size) { - .byte => unreachable, // Zig does not support 8-bit floats - .hword => return lower.fail("TODO: lowerMir pseudo_store_rm support 16-bit floats", .{}), - .word => .fsw, - .dword => .fsd, - }, - .vector => switch (dest_size) { - .byte => .vse8v, - .hword => .vse16v, - .word => .vse32v, - .dword => .vse64v, - }, - }; - - switch (src_reg_class) { - .int, .float => { - try lower.emit(mnem, &.{ - .{ .reg = frame_loc.base }, - .{ .reg = rm.r }, - .{ .imm = Immediate.s(frame_loc.disp) }, - }); - }, - .vector => { - assert(frame_loc.disp == 0); - try lower.emit(mnem, &.{ - .{ .reg = rm.r }, - .{ .reg = frame_loc.base }, - .{ .reg = .zero }, - }); - }, - } - }, - else => unreachable, - } - }, - - .pseudo_mv => { - const rr = inst.data.rr; - - const dst_class = rr.rd.class(); - const src_class = rr.rs.class(); - - switch (src_class) { - .float => switch (dst_class) { - .float => { - try lower.emit(if (lower.hasFeature(.d)) .fsgnjnd else .fsgnjns, &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - .{ .reg = rr.rs }, - }); - }, - .int, .vector => return lower.fail("TODO: lowerMir pseudo_mv float -> {s}", .{@tagName(dst_class)}), - }, - .int => switch (dst_class) { - .int => { - try lower.emit(.addi, &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - .{ .imm = Immediate.s(0) }, - }); - }, - .vector => { - try lower.emit(.vmvvx, &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - .{ .reg = .x0 }, - }); - }, - .float => return lower.fail("TODO: lowerMir pseudo_mv int -> {s}", .{@tagName(dst_class)}), - }, - .vector => switch (dst_class) { - .int => { - try lower.emit(.vadcvv, &.{ - .{ .reg = rr.rd }, - .{ .reg = .zero }, - .{ .reg = rr.rs }, - }); - }, - .float, .vector => return lower.fail("TODO: lowerMir pseudo_mv vector -> {s}", .{@tagName(dst_class)}), - }, - } - }, - - .pseudo_j => { - const j_type = inst.data.j_type; - try lower.emit(.jal, &.{ - .{ .reg = j_type.rd }, - .{ .imm = lower.reloc(.{ .inst = j_type.inst }) }, - }); - }, - - .pseudo_spill_regs => try lower.pushPopRegList(true, inst.data.reg_list), - .pseudo_restore_regs => try lower.pushPopRegList(false, inst.data.reg_list), - - .pseudo_load_symbol => { - const payload = inst.data.reloc; - const dst_reg = payload.register; - assert(dst_reg.class() == .int); - - try lower.emit(.lui, &.{ - .{ .reg = dst_reg }, - .{ .imm = lower.reloc(.{ - .load_symbol_reloc = .{ - .atom_index = payload.atom_index, - .sym_index = payload.sym_index, - }, - }) }, - }); - - // the reloc above implies this one - try lower.emit(.addi, &.{ - .{ .reg = dst_reg }, - .{ .reg = dst_reg }, - .{ .imm = Immediate.s(0) }, - }); - }, - - .pseudo_load_tlv => { - const payload = inst.data.reloc; - const dst_reg = payload.register; - assert(dst_reg.class() == .int); - - try lower.emit(.lui, &.{ - .{ .reg = dst_reg }, - .{ .imm = lower.reloc(.{ - .load_tlv_reloc = .{ - .atom_index = payload.atom_index, - .sym_index = payload.sym_index, - }, - }) }, - }); - - try lower.emit(.add, &.{ - .{ .reg = dst_reg }, - .{ .reg = dst_reg }, - .{ .reg = .tp }, - }); - - try lower.emit(.addi, &.{ - .{ .reg = dst_reg }, - .{ .reg = dst_reg }, - .{ .imm = Immediate.s(0) }, - }); - }, - - .pseudo_lea_rm => { - const rm = inst.data.rm; - assert(rm.r.class() == .int); - - const frame: Mir.FrameLoc = if (options.allow_frame_locs) - rm.m.toFrameLoc(lower.mir) - else - .{ .base = .s0, .disp = 0 }; - - try lower.emit(.addi, &.{ - .{ .reg = rm.r }, - .{ .reg = frame.base }, - .{ .imm = Immediate.s(frame.disp) }, - }); - }, - - .pseudo_compare => { - const compare = inst.data.compare; - const op = compare.op; - - const rd = compare.rd; - const rs1 = compare.rs1; - const rs2 = compare.rs2; - - const class = rs1.class(); - const ty = compare.ty; - const size = std.math.ceilPowerOfTwo(u64, ty.bitSize(zcu)) catch { - return lower.fail("pseudo_compare size {}", .{ty.bitSize(zcu)}); - }; - - const is_unsigned = ty.isUnsignedInt(zcu); - const less_than: Mnemonic = if (is_unsigned) .sltu else .slt; - - switch (class) { - .int => switch (op) { - .eq => { - try lower.emit(.xor, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - - try lower.emit(.sltiu, &.{ - .{ .reg = rd }, - .{ .reg = rd }, - .{ .imm = Immediate.s(1) }, - }); - }, - .neq => { - try lower.emit(.xor, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - - try lower.emit(.sltu, &.{ - .{ .reg = rd }, - .{ .reg = .zero }, - .{ .reg = rd }, - }); - }, - .gt => { - try lower.emit(less_than, &.{ - .{ .reg = rd }, - .{ .reg = rs2 }, - .{ .reg = rs1 }, - }); - }, - .gte => { - try lower.emit(less_than, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - try lower.emit(.xori, &.{ - .{ .reg = rd }, - .{ .reg = rd }, - .{ .imm = Immediate.s(1) }, - }); - }, - .lt => { - try lower.emit(less_than, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - }, - .lte => { - try lower.emit(less_than, &.{ - .{ .reg = rd }, - .{ .reg = rs2 }, - .{ .reg = rs1 }, - }); - - try lower.emit(.xori, &.{ - .{ .reg = rd }, - .{ .reg = rd }, - .{ .imm = Immediate.s(1) }, - }); - }, - }, - .float => switch (op) { - // eq - .eq => { - try lower.emit(if (size == 64) .feqd else .feqs, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - }, - // !(eq) - .neq => { - try lower.emit(if (size == 64) .feqd else .feqs, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - try lower.emit(.xori, &.{ - .{ .reg = rd }, - .{ .reg = rd }, - .{ .imm = Immediate.s(1) }, - }); - }, - .lt => { - try lower.emit(if (size == 64) .fltd else .flts, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - }, - .lte => { - try lower.emit(if (size == 64) .fled else .fles, &.{ - .{ .reg = rd }, - .{ .reg = rs1 }, - .{ .reg = rs2 }, - }); - }, - .gt => { - try lower.emit(if (size == 64) .fltd else .flts, &.{ - .{ .reg = rd }, - .{ .reg = rs2 }, - .{ .reg = rs1 }, - }); - }, - .gte => { - try lower.emit(if (size == 64) .fled else .fles, &.{ - .{ .reg = rd }, - .{ .reg = rs2 }, - .{ .reg = rs1 }, - }); - }, - }, - .vector => return lower.fail("TODO: lowerMir pseudo_cmp vector", .{}), - } - }, - - .pseudo_not => { - const rr = inst.data.rr; - assert(rr.rs.class() == .int and rr.rd.class() == .int); - - // mask out any other bits that aren't the boolean - try lower.emit(.andi, &.{ - .{ .reg = rr.rs }, - .{ .reg = rr.rs }, - .{ .imm = Immediate.s(1) }, - }); - - try lower.emit(.sltiu, &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - .{ .imm = Immediate.s(1) }, - }); - }, - - .pseudo_extern_fn_reloc => { - const inst_reloc = inst.data.reloc; - const link_reg = inst_reloc.register; - - try lower.emit(.auipc, &.{ - .{ .reg = link_reg }, - .{ .imm = lower.reloc( - .{ .call_extern_fn_reloc = .{ - .atom_index = inst_reloc.atom_index, - .sym_index = inst_reloc.sym_index, - } }, - ) }, - }); - - try lower.emit(.jalr, &.{ - .{ .reg = link_reg }, - .{ .reg = link_reg }, - .{ .imm = Immediate.s(0) }, - }); - }, - } - - return .{ - .insts = lower.result_insts[0..lower.result_insts_len], - .relocs = lower.result_relocs[0..lower.result_relocs_len], - }; -} - -fn generic(lower: *Lower, inst: Mir.Inst) Error!void { - const mnemonic = inst.tag; - try lower.emit(mnemonic, switch (inst.data) { - .none => &.{}, - .u_type => |u| &.{ - .{ .reg = u.rd }, - .{ .imm = u.imm20 }, - }, - .i_type => |i| &.{ - .{ .reg = i.rd }, - .{ .reg = i.rs1 }, - .{ .imm = i.imm12 }, - }, - .rr => |rr| &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - }, - .b_type => |b| &.{ - .{ .reg = b.rs1 }, - .{ .reg = b.rs2 }, - .{ .imm = lower.reloc(.{ .inst = b.inst }) }, - }, - .r_type => |r| &.{ - .{ .reg = r.rd }, - .{ .reg = r.rs1 }, - .{ .reg = r.rs2 }, - }, - .csr => |csr| &.{ - .{ .csr = csr.csr }, - .{ .reg = csr.rs1 }, - .{ .reg = csr.rd }, - }, - .amo => |amo| &.{ - .{ .reg = amo.rd }, - .{ .reg = amo.rs1 }, - .{ .reg = amo.rs2 }, - .{ .barrier = amo.rl }, - .{ .barrier = amo.aq }, - }, - .fence => |fence| &.{ - .{ .barrier = fence.succ }, - .{ .barrier = fence.pred }, - }, - else => return lower.fail("TODO: generic lower {s}", .{@tagName(inst.data)}), - }); -} - -fn emit(lower: *Lower, mnemonic: Mnemonic, ops: []const Instruction.Operand) !void { - const lir = encoding.Lir.fromMnem(mnemonic); - const inst = Instruction.fromLir(lir, ops); - - lower.result_insts[lower.result_insts_len] = inst; - lower.result_insts_len += 1; -} - -fn reloc(lower: *Lower, target: Reloc.Target) Immediate { - lower.result_relocs[lower.result_relocs_len] = .{ - .lowered_inst_index = lower.result_insts_len, - .target = target, - }; - lower.result_relocs_len += 1; - return Immediate.s(0); -} - -fn pushPopRegList(lower: *Lower, comptime spilling: bool, reg_list: Mir.RegisterList) !void { - var it = reg_list.iterator(.{ .direction = .forward }); - - var reg_i: u31 = 0; - while (it.next()) |i| { - const frame = lower.mir.frame_locs.get(@intFromEnum(bits.FrameIndex.spill_frame)); - const reg = abi.Registers.all_preserved[i]; - - const reg_class = reg.class(); - const load_inst: Mnemonic, const store_inst: Mnemonic = switch (reg_class) { - .int => .{ .ld, .sd }, - .float => .{ .fld, .fsd }, - .vector => unreachable, - }; - - if (spilling) { - try lower.emit(store_inst, &.{ - .{ .reg = frame.base }, - .{ .reg = abi.Registers.all_preserved[i] }, - .{ .imm = Immediate.s(frame.disp + reg_i) }, - }); - } else { - try lower.emit(load_inst, &.{ - .{ .reg = abi.Registers.all_preserved[i] }, - .{ .reg = frame.base }, - .{ .imm = Immediate.s(frame.disp + reg_i) }, - }); - } - - reg_i += 8; - } -} - -pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { - @branchHint(.cold); - assert(lower.err_msg == null); - lower.err_msg = try ErrorMsg.create(lower.allocator, lower.src_loc, format, args); - return error.LowerFail; -} - -fn hasFeature(lower: *Lower, feature: std.Target.riscv.Feature) bool { - return lower.pt.zcu.getTarget().cpu.has(.riscv, feature); -} - -const Lower = @This(); -const std = @import("std"); -const assert = std.debug.assert; -const log = std.log.scoped(.lower); - -const Allocator = std.mem.Allocator; -const ErrorMsg = Zcu.ErrorMsg; - -const link = @import("../../link.zig"); -const Air = @import("../../Air.zig"); -const Zcu = @import("../../Zcu.zig"); - -const Mir = @import("Mir.zig"); -const abi = @import("abi.zig"); -const bits = @import("bits.zig"); -const encoding = @import("encoding.zig"); - -const Mnemonic = @import("mnem.zig").Mnemonic; -const Immediate = bits.Immediate; -const Instruction = encoding.Instruction; diff --git a/src/arch/riscv64/Mir.zig b/src/arch/riscv64/Mir.zig deleted file mode 100644 index 204ab6a3c06226635b1619b60c205f02f8447884..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/Mir.zig +++ /dev/null @@ -1,248 +0,0 @@ -//! Machine Intermediate Representation. -//! This data is produced by CodeGen.zig - -instructions: std.MultiArrayList(Inst).Slice, -frame_locs: std.MultiArrayList(FrameLoc).Slice, - -pub const Inst = struct { - tag: Mnemonic, - data: Data, - - pub const Index = u32; - - pub const Data = union(enum) { - none: void, - r_type: struct { - rd: Register, - rs1: Register, - rs2: Register, - }, - i_type: struct { - rd: Register, - rs1: Register, - imm12: Immediate, - }, - s_type: struct { - rs1: Register, - rs2: Register, - imm5: Immediate, - imm7: Immediate, - }, - b_type: struct { - rs1: Register, - rs2: Register, - inst: Inst.Index, - }, - u_type: struct { - rd: Register, - imm20: Immediate, - }, - j_type: struct { - rd: Register, - inst: Inst.Index, - }, - rm: struct { - r: Register, - m: Memory, - }, - reg_list: Mir.RegisterList, - reg: Register, - rr: struct { - rd: Register, - rs: Register, - }, - compare: struct { - rd: Register, - rs1: Register, - rs2: Register, - op: enum { - eq, - neq, - gt, - gte, - lt, - lte, - }, - ty: Type, - }, - reloc: struct { - register: Register, - atom_index: u32, - sym_index: u32, - }, - fence: struct { - pred: Barrier, - succ: Barrier, - }, - amo: struct { - rd: Register, - rs1: Register, - rs2: Register, - aq: Barrier, - rl: Barrier, - }, - csr: struct { - csr: CSR, - rs1: Register, - rd: Register, - }, - pseudo_dbg_line_column: struct { - line: u32, - column: u32, - }, - }; - - pub fn format(inst: Inst, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Tag: {s}, Data: {s}", .{ @tagName(inst.tag), @tagName(inst.data) }); - } -}; - -pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { - mir.instructions.deinit(gpa); - mir.frame_locs.deinit(gpa); - mir.* = undefined; -} - -pub fn emit( - mir: Mir, - lf: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - atom_index: u32, - w: *std.Io.Writer, - debug_output: link.File.DebugInfoOutput, -) (codegen.CodeGenError || std.Io.Writer.Error)!void { - _ = atom_index; - const zcu = pt.zcu; - const comp = zcu.comp; - const gpa = comp.gpa; - const func = zcu.funcInfo(func_index); - const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?; - const nav = func.owner_nav; - const mod = zcu.navFileScope(nav).mod.?; - var e: Emit = .{ - .lower = .{ - .pt = pt, - .allocator = gpa, - .mir = mir, - .cc = fn_info.cc, - .src_loc = src_loc, - .output_mode = comp.config.output_mode, - .link_mode = comp.config.link_mode, - .pic = mod.pic, - }, - .bin_file = lf, - .debug_output = debug_output, - .w = w, - .prev_di_pc = 0, - .prev_di_line = func.lbrace_line, - .prev_di_column = func.lbrace_column, - }; - defer e.deinit(); - e.emitMir() catch |err| switch (err) { - error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, e.lower.err_msg.?), - error.InvalidInstruction => return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}), - else => |err1| return err1, - }; -} - -pub const FrameLoc = struct { - base: Register, - disp: i32, -}; - -pub const Barrier = enum(u4) { - // Fence - w = 0b0001, - r = 0b0010, - rw = 0b0011, - - // Amo - none, - aq, - rl, -}; - -pub const AmoOp = enum(u5) { - SWAP, - ADD, - AND, - OR, - XOR, - MAX, - MIN, -}; - -pub const FcvtOp = enum(u5) { - w = 0b00000, - wu = 0b00001, - l = 0b00010, - lu = 0b00011, -}; - -pub const LoadSymbolPayload = struct { - register: u32, - atom_index: u32, - sym_index: u32, -}; - -/// Used in conjunction with payload to transfer a list of used registers in a compact manner. -pub const RegisterList = struct { - bitset: BitSet = BitSet.initEmpty(), - - const BitSet = IntegerBitSet(32); - const Self = @This(); - - fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt { - for (registers, 0..) |cpreg, i| { - if (reg.id() == cpreg.id()) return @intCast(i); - } - unreachable; // register not in input register list! - } - - pub fn push(self: *Self, registers: []const Register, reg: Register) void { - const index = getIndexForReg(registers, reg); - self.bitset.set(index); - } - - pub fn isSet(self: Self, registers: []const Register, reg: Register) bool { - const index = getIndexForReg(registers, reg); - return self.bitset.isSet(index); - } - - pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) { - return self.bitset.iterator(options); - } - - pub fn count(self: Self) i32 { - return @intCast(self.bitset.count()); - } - - pub fn size(self: Self) i32 { - return @intCast(self.bitset.count() * 8); - } -}; - -const Mir = @This(); -const std = @import("std"); -const builtin = @import("builtin"); -const Type = @import("../../Type.zig"); -const bits = @import("bits.zig"); - -const assert = std.debug.assert; - -const Register = bits.Register; -const CSR = bits.CSR; -const Immediate = bits.Immediate; -const Memory = bits.Memory; -const FrameIndex = bits.FrameIndex; -const FrameAddr = @import("CodeGen.zig").FrameAddr; -const IntegerBitSet = std.bit_set.IntegerBitSet; -const Mnemonic = @import("mnem.zig").Mnemonic; - -const InternPool = @import("../../InternPool.zig"); -const Emit = @import("Emit.zig"); -const codegen = @import("../../codegen.zig"); -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); diff --git a/src/arch/riscv64/abi.zig b/src/arch/riscv64/abi.zig deleted file mode 100644 index 1a380eab905708b866f18a29762e1653cd0a521b..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/abi.zig +++ /dev/null @@ -1,353 +0,0 @@ -const std = @import("std"); -const bits = @import("bits.zig"); -const Register = bits.Register; -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const Type = @import("../../Type.zig"); -const InternPool = @import("../../InternPool.zig"); -const Zcu = @import("../../Zcu.zig"); -const assert = std.debug.assert; - -pub const Class = enum { memory, byval, integer, double_integer, fields }; - -pub fn classifyType(ty: Type, zcu: *Zcu) Class { - const target = zcu.getTarget(); - std.debug.assert(ty.hasRuntimeBitsIgnoreComptime(zcu)); - - const max_byval_size = target.ptrBitWidth() * 2; - switch (ty.zigTypeTag(zcu)) { - .@"struct" => { - const bit_size = ty.bitSize(zcu); - if (ty.containerLayout(zcu) == .@"packed") { - if (bit_size > max_byval_size) return .memory; - return .byval; - } - - if (target.cpu.has(.riscv, .d)) fields: { - var any_fp = false; - var field_count: usize = 0; - for (0..ty.structFieldCount(zcu)) |field_index| { - const field_ty = ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - if (field_ty.isRuntimeFloat()) - any_fp = true - else if (!field_ty.isAbiInt(zcu)) - break :fields; - field_count += 1; - if (field_count > 2) break :fields; - } - std.debug.assert(field_count > 0 and field_count <= 2); - if (any_fp) return .fields; - } - - // TODO this doesn't exactly match what clang produces but its better than nothing - if (bit_size > max_byval_size) return .memory; - if (bit_size > max_byval_size / 2) return .double_integer; - return .integer; - }, - .@"union" => { - const bit_size = ty.bitSize(zcu); - if (ty.containerLayout(zcu) == .@"packed") { - if (bit_size > max_byval_size) return .memory; - return .byval; - } - // TODO this doesn't exactly match what clang produces but its better than nothing - if (bit_size > max_byval_size) return .memory; - if (bit_size > max_byval_size / 2) return .double_integer; - return .integer; - }, - .bool => return .integer, - .float => return .byval, - .int, .@"enum", .error_set => { - const bit_size = ty.bitSize(zcu); - if (bit_size > max_byval_size) return .memory; - return .byval; - }, - .vector => { - const bit_size = ty.bitSize(zcu); - if (bit_size > max_byval_size) return .memory; - return .integer; - }, - .optional => { - std.debug.assert(ty.isPtrLikeOptional(zcu)); - return .byval; - }, - .pointer => { - std.debug.assert(!ty.isSlice(zcu)); - return .byval; - }, - .error_union, - .frame, - .@"anyframe", - .noreturn, - .void, - .type, - .comptime_float, - .comptime_int, - .undefined, - .null, - .@"fn", - .@"opaque", - .enum_literal, - .array, - => unreachable, - } -} - -pub const SystemClass = enum { integer, float, memory, none }; - -/// There are a maximum of 8 possible return slots. Returned values are in -/// the beginning of the array; unused slots are filled with .none. -pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass { - var result = [1]SystemClass{.none} ** 8; - const memory_class = [_]SystemClass{ - .memory, .none, .none, .none, - .none, .none, .none, .none, - }; - switch (ty.zigTypeTag(zcu)) { - .bool, .void, .noreturn => { - result[0] = .integer; - return result; - }, - .pointer => switch (ty.ptrSize(zcu)) { - .slice => { - result[0] = .integer; - result[1] = .integer; - return result; - }, - else => { - result[0] = .integer; - return result; - }, - }, - .optional => { - if (ty.isPtrLikeOptional(zcu)) { - result[0] = .integer; - return result; - } - return memory_class; - }, - .int, .@"enum", .error_set => { - const int_bits = ty.intInfo(zcu).bits; - if (int_bits <= 64) { - result[0] = .integer; - return result; - } - if (int_bits <= 128) { - result[0] = .integer; - result[1] = .integer; - return result; - } - unreachable; // support > 128 bit int arguments - }, - .float => { - const target = zcu.getTarget(); - - const float_bits = ty.floatBits(target); - const float_reg_size: u32 = if (target.cpu.has(.riscv, .d)) 64 else 32; - if (float_bits <= float_reg_size) { - result[0] = .float; - return result; - } - unreachable; // support split float args - }, - .error_union => { - const payload_ty = ty.errorUnionPayload(zcu); - const payload_bits = payload_ty.bitSize(zcu); - - // the error union itself - result[0] = .integer; - - // anyerror!void can fit into one register - if (payload_bits == 0) return result; - - return memory_class; - }, - .@"struct", .@"union" => { - const layout = ty.containerLayout(zcu); - const ty_size = ty.abiSize(zcu); - - if (layout == .@"packed") { - assert(ty_size <= 16); - result[0] = .integer; - if (ty_size > 8) result[1] = .integer; - return result; - } - - return memory_class; - }, - .array => { - const ty_size = ty.abiSize(zcu); - if (ty_size <= 8) { - result[0] = .integer; - return result; - } - if (ty_size <= 16) { - result[0] = .integer; - result[1] = .integer; - return result; - } - return memory_class; - }, - .vector => { - // we pass vectors through integer registers if they are small enough to fit. - const vec_bits = ty.totalVectorBits(zcu); - if (vec_bits <= 64) { - result[0] = .integer; - return result; - } - // we should pass vector registers of size <= 128 through 2 integer registers - // but we haven't implemented seperating vector registers into register_pairs - return memory_class; - }, - else => |bad_ty| std.debug.panic("classifySystem {s}", .{@tagName(bad_ty)}), - } -} - -fn classifyStruct( - result: *[8]Class, - byte_offset: *u64, - loaded_struct: InternPool.LoadedStructType, - zcu: *Zcu, -) void { - const ip = &zcu.intern_pool; - var field_it = loaded_struct.iterateRuntimeOrder(ip); - - while (field_it.next()) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); - const field_align = loaded_struct.fieldAlign(ip, field_index); - byte_offset.* = std.mem.alignForward( - u64, - byte_offset.*, - field_align.toByteUnits() orelse field_ty.abiAlignment(zcu).toByteUnits().?, - ); - if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { - if (field_loaded_struct.layout != .@"packed") { - classifyStruct(result, byte_offset, field_loaded_struct, zcu); - continue; - } - } - const field_class = std.mem.sliceTo(&classifySystem(field_ty, zcu), .none); - const field_size = field_ty.abiSize(zcu); - - combine: { - const result_class = &result[@intCast(byte_offset.* / 8)]; - if (result_class.* == field_class[0]) { - break :combine; - } - - if (result_class.* == .none) { - result_class.* = field_class[0]; - break :combine; - } - assert(field_class[0] != .none); - - // "If one of the classes is MEMORY, the result is the MEMORY class." - if (result_class.* == .memory or field_class[0] == .memory) { - result_class.* = .memory; - break :combine; - } - - // "If one of the classes is INTEGER, the result is the INTEGER." - if (result_class.* == .integer or field_class[0] == .integer) { - result_class.* = .integer; - break :combine; - } - - result_class.* = .integer; - } - @memcpy(result[@intCast(byte_offset.* / 8 + 1)..][0 .. field_class.len - 1], field_class[1..]); - byte_offset.* += field_size; - } -} - -const allocatable_registers = Registers.Integer.all_regs ++ Registers.Float.all_regs ++ Registers.Vector.all_regs; -pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); - -const RegisterBitSet = RegisterManager.RegisterBitSet; - -pub const RegisterClass = enum { - int, - float, - vector, -}; - -pub const Registers = struct { - pub const all_preserved = Integer.callee_preserved_regs ++ Float.callee_preserved_regs; - - pub const Integer = struct { - // zig fmt: off - pub const general_purpose = initRegBitSet(0, callee_preserved_regs.len); - pub const function_arg = initRegBitSet(callee_preserved_regs.len, function_arg_regs.len); - pub const function_ret = initRegBitSet(callee_preserved_regs.len, function_ret_regs.len); - pub const temporary = initRegBitSet(callee_preserved_regs.len + function_arg_regs.len, temporary_regs.len); - // zig fmt: on - - pub const callee_preserved_regs = [_]Register{ - // .s0 is omitted to be used as the frame pointer register - .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11, - }; - - pub const function_arg_regs = [_]Register{ - .a0, .a1, .a2, .a3, .a4, .a5, .a6, .a7, - }; - - pub const function_ret_regs = [_]Register{ - .a0, .a1, - }; - - pub const temporary_regs = [_]Register{ - .t0, .t1, .t2, .t3, .t4, .t5, .t6, - }; - - pub const all_regs = callee_preserved_regs ++ function_arg_regs ++ temporary_regs; - }; - - pub const Float = struct { - // zig fmt: off - pub const general_purpose = initRegBitSet(Integer.all_regs.len, callee_preserved_regs.len); - pub const function_arg = initRegBitSet(Integer.all_regs.len + callee_preserved_regs.len, function_arg_regs.len); - pub const function_ret = initRegBitSet(Integer.all_regs.len + callee_preserved_regs.len, function_ret_regs.len); - pub const temporary = initRegBitSet(Integer.all_regs.len + callee_preserved_regs.len + function_arg_regs.len, temporary_regs.len); - // zig fmt: on - - pub const callee_preserved_regs = [_]Register{ - .fs0, .fs1, .fs2, .fs3, .fs4, .fs5, .fs6, .fs7, .fs8, .fs9, .fs10, .fs11, - }; - - pub const function_arg_regs = [_]Register{ - .fa0, .fa1, .fa2, .fa3, .fa4, .fa5, .fa6, .fa7, - }; - - pub const function_ret_regs = [_]Register{ - .fa0, .fa1, - }; - - pub const temporary_regs = [_]Register{ - .ft0, .ft1, .ft2, .ft3, .ft4, .ft5, .ft6, .ft7, .ft8, .ft9, .ft10, .ft11, - }; - - pub const all_regs = callee_preserved_regs ++ function_arg_regs ++ temporary_regs; - }; - - pub const Vector = struct { - pub const general_purpose = initRegBitSet(Integer.all_regs.len + Float.all_regs.len, all_regs.len); - - // zig fmt: off - pub const all_regs = [_]Register{ - .v0, .v1, .v2, .v3, .v4, .v5, .v6, .v7, - .v8, .v9, .v10, .v11, .v12, .v13, .v14, .v15, - .v16, .v17, .v18, .v19, .v20, .v21, .v22, .v23, - .v24, .v25, .v26, .v27, .v28, .v29, .v30, .v31, - }; - // zig fmt: on - }; -}; - -fn initRegBitSet(start: usize, length: usize) RegisterBitSet { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = start, - .end = start + length, - }, true); - return set; -} diff --git a/src/arch/riscv64/bits.zig b/src/arch/riscv64/bits.zig deleted file mode 100644 index 94c64dfc98de50e7ecb8183a2008c7014e222ebd..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/bits.zig +++ /dev/null @@ -1,290 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; -const Target = std.Target; - -const Zcu = @import("../../Zcu.zig"); -const Mir = @import("Mir.zig"); -const abi = @import("abi.zig"); - -pub const Memory = struct { - base: Base, - mod: Mod, - - pub const Base = union(enum) { - reg: Register, - frame: FrameIndex, - }; - - pub const Mod = struct { - size: Size, - unsigned: bool, - disp: i32 = 0, - }; - - pub const Size = enum(u4) { - /// Byte, 1 byte - byte, - /// Half word, 2 bytes - hword, - /// Word, 4 bytes - word, - /// Double word, 8 Bytes - dword, - - pub fn fromByteSize(size: u64) Size { - return switch (size) { - 1...1 => .byte, - 2...2 => .hword, - 3...4 => .word, - 5...8 => .dword, - else => std.debug.panic("fromByteSize {}", .{size}), - }; - } - - pub fn fromBitSize(bit_size: u64) Size { - return switch (bit_size) { - 8 => .byte, - 16 => .hword, - 32 => .word, - 64 => .dword, - else => unreachable, - }; - } - - pub fn bitSize(s: Size) u64 { - return switch (s) { - .byte => 8, - .hword => 16, - .word => 32, - .dword => 64, - }; - } - }; - - /// Asserts `mem` can be represented as a `FrameLoc`. - pub fn toFrameLoc(mem: Memory, mir: Mir) Mir.FrameLoc { - const offset: i32 = mem.mod.disp; - - switch (mem.base) { - .reg => |reg| { - return .{ - .base = reg, - .disp = offset, - }; - }, - .frame => |index| { - const base_loc = mir.frame_locs.get(@intFromEnum(index)); - return .{ - .base = base_loc.base, - .disp = base_loc.disp + offset, - }; - }, - } - } -}; - -pub const Immediate = union(enum) { - signed: i32, - unsigned: u64, - - pub fn u(x: u64) Immediate { - return .{ .unsigned = x }; - } - - pub fn s(x: i32) Immediate { - return .{ .signed = x }; - } - - pub fn asSigned(imm: Immediate, bit_size: u64) i64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(i8, @intCast(x)), - 16 => @as(i16, @intCast(x)), - 32, 64 => x, - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), - 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), - 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), - 64 => @bitCast(x), - else => unreachable, - }, - }; - } - - pub fn asBits(imm: Immediate, comptime T: type) T { - const int_info = @typeInfo(T).int; - if (int_info.signedness != .unsigned) @compileError("Immediate.asBits needs unsigned T"); - return switch (imm) { - .signed => |x| @bitCast(@as(std.meta.Int(.signed, int_info.bits), @intCast(x))), - .unsigned => |x| @intCast(x), - }; - } -}; - -pub const CSR = enum(u12) { - vl = 0xC20, - vtype = 0xC21, - vlenb = 0xC22, -}; - -pub const Register = enum(u8) { - // zig fmt: off - - // base extension registers - - zero, // zero - ra, // return address. caller saved - sp, // stack pointer. callee saved. - gp, // global pointer - tp, // thread pointer - t0, t1, t2, // temporaries. caller saved. - s0, // s0/fp, callee saved. - s1, // callee saved. - a0, a1, // fn args/return values. caller saved. - a2, a3, a4, a5, a6, a7, // fn args. caller saved. - s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, // saved registers. callee saved. - t3, t4, t5, t6, // caller saved - - x0, x1, x2, x3, x4, x5, x6, x7, - x8, x9, x10, x11, x12, x13, x14, x15, - x16, x17, x18, x19, x20, x21, x22, x23, - x24, x25, x26, x27, x28, x29, x30, x31, - - - // F extension registers - - ft0, ft1, ft2, ft3, ft4, ft5, ft6, ft7, // float temporaries. caller saved. - fs0, fs1, // float saved. callee saved. - fa0, fa1, // float arg/ret. caller saved. - fa2, fa3, fa4, fa5, fa6, fa7, // float arg. called saved. - fs2, fs3, fs4, fs5, fs6, fs7, fs8, fs9, fs10, fs11, // float saved. callee saved. - ft8, ft9, ft10, ft11, // foat temporaries. calller saved. - - // this register is accessed only through API instructions instead of directly - // fcsr, - - f0, f1, f2, f3, f4, f5, f6, f7, - f8, f9, f10, f11, f12, f13, f14, f15, - f16, f17, f18, f19, f20, f21, f22, f23, - f24, f25, f26, f27, f28, f29, f30, f31, - - - // V extension registers - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31, - - // zig fmt: on - - /// in RISC-V registers are stored as 5 bit IDs and a register can have - /// two names. Example being `zero` and `x0` are the same register and have the - /// same ID, but are two different entries in the enum. We store floating point - /// registers in the same enum. RISC-V uses the same IDs for `f0` and `x0` by - /// infering which register is being talked about given the instruction it's in. - /// - /// The goal of this function is to return the same ID for `zero` and `x0` but two - /// seperate IDs for `x0` and `f0`. We will assume that each register set has 32 registers - /// and is repeated twice, once for the named version, once for the number version. - pub fn id(reg: Register) std.math.IntFittingRange(0, @typeInfo(Register).@"enum".fields.len) { - const base = switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => @intFromEnum(Register.zero), - @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => @intFromEnum(Register.ft0), - @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => @intFromEnum(Register.v0), - else => unreachable, - // zig fmt: on - }; - - return @intCast(base + reg.encodeId()); - } - - pub fn encodeId(reg: Register) u5 { - return @truncate(@intFromEnum(reg)); - } - - pub fn dwarfNum(reg: Register) u8 { - return reg.id(); - } - - pub fn bitSize(reg: Register, zcu: *const Zcu) u32 { - return switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => 64, - @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => if (zcu.getTarget().cpu.has(.riscv, .d)) 64 else 32, - @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => 256, // TODO: look at suggestVectorSize - else => unreachable, - // zig fmt: on - }; - } - - pub fn class(reg: Register) abi.RegisterClass { - return switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => .int, - @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => .float, - @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => .vector, - else => unreachable, - // zig fmt: on - }; - } -}; - -pub const FrameIndex = enum(u32) { - /// This index refers to the return address. - ret_addr, - /// This index refers to the frame pointer. - base_ptr, - /// This index refers to the entire stack frame. - stack_frame, - /// This index referes to where in the stack frame the args are spilled to. - args_frame, - /// This index referes to a frame dedicated to setting up args for function called - /// in this function. Useful for aligning args separately. - call_frame, - /// This index referes to the frame where callee saved registers are spilled and restored from. - spill_frame, - /// Other indices are used for local variable stack slots - _, - - pub const named_count = @typeInfo(FrameIndex).@"enum".fields.len; - - pub fn isNamed(fi: FrameIndex) bool { - return @intFromEnum(fi) < named_count; - } -}; - -/// A linker symbol not yet allocated in VM. -pub const Symbol = struct { - /// Index of the containing atom. - atom_index: u32, - /// Index into the linker's symbol table. - sym_index: u32, -}; - -pub const VType = packed struct(u8) { - vlmul: VlMul, - vsew: VSew, - vta: bool, - vma: bool, -}; - -const VSew = enum(u3) { - @"8" = 0b000, - @"16" = 0b001, - @"32" = 0b010, - @"64" = 0b011, -}; - -const VlMul = enum(u3) { - mf8 = 0b101, - mf4 = 0b110, - mf2 = 0b111, - m1 = 0b000, - m2 = 0b001, - m4 = 0b010, - m8 = 0b011, -}; diff --git a/src/arch/riscv64/encoding.zig b/src/arch/riscv64/encoding.zig deleted file mode 100644 index 109e1d08f8cef5ace0e88a78ecd426536c90f34f..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/encoding.zig +++ /dev/null @@ -1,729 +0,0 @@ -//! This file is responsible for going from MIR, which is emitted by CodeGen -//! and converting it into Instructions, which can be used as needed. -//! -//! Here we encode how mnemonics relate to opcodes and where their operands go. - -/// Lower Instruction Representation -/// -/// This format encodes a specific instruction, however it's still abstracted -/// away from the true encoding it'll be in. It's meant to make the process of -/// indicating unique encoding data easier. -pub const Lir = struct { - opcode: OpCode, - format: Format, - data: Data, - - pub const Format = enum { - R, - I, - S, - B, - U, - J, - extra, - }; - - const Data = union(enum) { - none, - f: struct { funct3: u3 }, - ff: struct { - funct3: u3, - funct7: u7, - }, - sh: struct { - typ: u6, - funct3: u3, - has_5: bool, - }, - - fmt: struct { - funct5: u5, - rm: u3, - fmt: FpFmt, - }, - fcvt: struct { - funct5: u5, - rm: u3, - fmt: FpFmt, - width: Mir.FcvtOp, - }, - - vecls: struct { - width: VecWidth, - umop: Umop, - vm: bool, - mop: Mop, - mew: bool, - nf: u3, - }, - vecmath: struct { - vm: bool, - funct6: u6, - funct3: VecType, - }, - - amo: struct { - funct5: u5, - width: AmoWidth, - }, - fence: struct { - funct3: u3, - fm: FenceMode, - }, - - /// the mnemonic has some special properities that can't be handled in a generic fashion - extra: Mnemonic, - }; - - const OpCode = enum(u7) { - LOAD = 0b0000011, - LOAD_FP = 0b0000111, - MISC_MEM = 0b0001111, - OP_IMM = 0b0010011, - AUIPC = 0b0010111, - OP_IMM_32 = 0b0011011, - STORE = 0b0100011, - STORE_FP = 0b0100111, - AMO = 0b0101111, - OP_V = 0b1010111, - OP = 0b0110011, - OP_32 = 0b0111011, - LUI = 0b0110111, - MADD = 0b1000011, - MSUB = 0b1000111, - NMSUB = 0b1001011, - NMADD = 0b1001111, - OP_FP = 0b1010011, - OP_IMM_64 = 0b1011011, - BRANCH = 0b1100011, - JALR = 0b1100111, - JAL = 0b1101111, - SYSTEM = 0b1110011, - OP_64 = 0b1111011, - NONE = 0b00000000, - }; - - const FpFmt = enum(u2) { - /// 32-bit single-precision - S = 0b00, - /// 64-bit double-precision - D = 0b01, - - // H = 0b10, unused in the G extension - - /// 128-bit quad-precision - Q = 0b11, - }; - - const AmoWidth = enum(u3) { - W = 0b010, - D = 0b011, - }; - - const FenceMode = enum(u4) { - none = 0b0000, - tso = 0b1000, - }; - - const Mop = enum(u2) { - // zig fmt: off - unit = 0b00, - unord = 0b01, - stride = 0b10, - ord = 0b11, - // zig fmt: on - }; - - const Umop = enum(u5) { - // zig fmt: off - unit = 0b00000, - whole = 0b01000, - mask = 0b01011, - fault = 0b10000, - // zig fmt: on - }; - - const VecWidth = enum(u3) { - // zig fmt: off - @"8" = 0b000, - @"16" = 0b101, - @"32" = 0b110, - @"64" = 0b111, - // zig fmt: on - }; - - const VecType = enum(u3) { - OPIVV = 0b000, - OPFVV = 0b001, - OPMVV = 0b010, - OPIVI = 0b011, - OPIVX = 0b100, - OPFVF = 0b101, - OPMVX = 0b110, - }; - - pub fn fromMnem(mnem: Mnemonic) Lir { - return switch (mnem) { - // zig fmt: off - - // OP - .add => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000000 } } }, - .sub => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0100000 } } }, - - .@"and" => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b111, .funct7 = 0b0000000 } } }, - .@"or" => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b110, .funct7 = 0b0000000 } } }, - .xor => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b100, .funct7 = 0b0000000 } } }, - - .sltu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b011, .funct7 = 0b0000000 } } }, - .slt => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b010, .funct7 = 0b0000000 } } }, - - .mul => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000001 } } }, - .mulh => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0000001 } } }, - .mulhsu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b010, .funct7 = 0b0000001 } } }, - .mulhu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b011, .funct7 = 0b0000001 } } }, - - .div => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b100, .funct7 = 0b0000001 } } }, - .divu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000001 } } }, - - .rem => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b110, .funct7 = 0b0000001 } } }, - .remu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b111, .funct7 = 0b0000001 } } }, - - .sll => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0000000 } } }, - .srl => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000000 } } }, - .sra => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0100000 } } }, - - - // OP_IMM - - .addi => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b000 } } }, - .andi => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b111 } } }, - .xori => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b100 } } }, - - .sltiu => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b011 } } }, - - .slli => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b001, .has_5 = true } } }, - .srli => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b101, .has_5 = true } } }, - .srai => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = true } } }, - - .clz => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, - .cpop => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, - - // OP_IMM_32 - - .slliw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b001, .has_5 = false } } }, - .srliw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b101, .has_5 = false } } }, - .sraiw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = false } } }, - - .clzw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, - .cpopw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, - - // OP_32 - - .addw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000000 } } }, - .subw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0100000 } } }, - .mulw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000001 } } }, - - .divw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b100, .funct7 = 0b0000001 } } }, - .divuw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000001 } } }, - - .remw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b110, .funct7 = 0b0000001 } } }, - .remuw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b111, .funct7 = 0b0000001 } } }, - - .sllw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0000000 } } }, - .srlw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000000 } } }, - .sraw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0100000 } } }, - - - // OP_FP - - .fadds => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00000, .fmt = .S, .rm = 0b111 } } }, - .faddd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00000, .fmt = .D, .rm = 0b111 } } }, - - .fsubs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00001, .fmt = .S, .rm = 0b111 } } }, - .fsubd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00001, .fmt = .D, .rm = 0b111 } } }, - - .fmuls => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00010, .fmt = .S, .rm = 0b111 } } }, - .fmuld => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00010, .fmt = .D, .rm = 0b111 } } }, - - .fdivs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00011, .fmt = .S, .rm = 0b111 } } }, - .fdivd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00011, .fmt = .D, .rm = 0b111 } } }, - - .fmins => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .S, .rm = 0b000 } } }, - .fmind => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .D, .rm = 0b000 } } }, - - .fmaxs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .S, .rm = 0b001 } } }, - .fmaxd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .D, .rm = 0b001 } } }, - - .fsqrts => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b01011, .fmt = .S, .rm = 0b111 } } }, - .fsqrtd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b01011, .fmt = .D, .rm = 0b111 } } }, - - .fles => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .S, .rm = 0b000 } } }, - .fled => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .D, .rm = 0b000 } } }, - - .flts => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .S, .rm = 0b001 } } }, - .fltd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .D, .rm = 0b001 } } }, - - .feqs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .S, .rm = 0b010 } } }, - .feqd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .D, .rm = 0b010 } } }, - - .fsgnjns => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .S, .rm = 0b000 } } }, - .fsgnjnd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .D, .rm = 0b000 } } }, - - .fsgnjxs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .S, .rm = 0b010 } } }, - .fsgnjxd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .D, .rm = 0b010 } } }, - - .fcvtws => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .w } } }, - .fcvtwus => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .wu } } }, - .fcvtls => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .l } } }, - .fcvtlus => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .lu } } }, - - .fcvtwd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .w } } }, - .fcvtwud => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .wu } } }, - .fcvtld => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .l } } }, - .fcvtlud => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .lu } } }, - - .fcvtsw => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .w } } }, - .fcvtswu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .wu } } }, - .fcvtsl => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .l } } }, - .fcvtslu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .lu } } }, - - .fcvtdw => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .w } } }, - .fcvtdwu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .wu } } }, - .fcvtdl => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .l } } }, - .fcvtdlu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .lu } } }, - - // LOAD - - .lb => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b000 } } }, - .lh => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b001 } } }, - .lw => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b010 } } }, - .ld => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b011 } } }, - .lbu => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b100 } } }, - .lhu => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b101 } } }, - .lwu => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b110 } } }, - - - // STORE - - .sb => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b000 } } }, - .sh => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b001 } } }, - .sw => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b010 } } }, - .sd => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b011 } } }, - - - // LOAD_FP - - .flw => .{ .opcode = .LOAD_FP, .format = .I, .data = .{ .f = .{ .funct3 = 0b010 } } }, - .fld => .{ .opcode = .LOAD_FP, .format = .I, .data = .{ .f = .{ .funct3 = 0b011 } } }, - - .vle8v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"8", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - .vle16v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"16", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - .vle32v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"32", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - .vle64v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"64", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - - - // STORE_FP - - .fsw => .{ .opcode = .STORE_FP, .format = .S, .data = .{ .f = .{ .funct3 = 0b010 } } }, - .fsd => .{ .opcode = .STORE_FP, .format = .S, .data = .{ .f = .{ .funct3 = 0b011 } } }, - - .vse8v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"8", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - .vse16v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"16", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - .vse32v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"32", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - .vse64v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"64", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, - - // JALR - - .jalr => .{ .opcode = .JALR, .format = .I, .data = .{ .f = .{ .funct3 = 0b000 } } }, - - - // LUI - - .lui => .{ .opcode = .LUI, .format = .U, .data = .{ .none = {} } }, - - - // AUIPC - - .auipc => .{ .opcode = .AUIPC, .format = .U, .data = .{ .none = {} } }, - - - // JAL - - .jal => .{ .opcode = .JAL, .format = .J, .data = .{ .none = {} } }, - - - // BRANCH - - .beq => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b000 } } }, - .bne => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b001 } } }, - - - // SYSTEM - - .ecall => .{ .opcode = .SYSTEM, .format = .extra, .data = .{ .extra = .ecall } }, - .ebreak => .{ .opcode = .SYSTEM, .format = .extra, .data = .{ .extra = .ebreak } }, - - .csrrs => .{ .opcode = .SYSTEM, .format = .I, .data = .{ .f = .{ .funct3 = 0b010 } } }, - - - // NONE - - .unimp => .{ .opcode = .NONE, .format = .extra, .data = .{ .extra = .unimp } }, - - - // MISC_MEM - - .fence => .{ .opcode = .MISC_MEM, .format = .I, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .none } } }, - .fencetso => .{ .opcode = .MISC_MEM, .format = .I, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .tso } } }, - - - // AMO - - .amoaddw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } }, - .amoswapw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } }, - .lrw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00010 } } }, - .scw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00011 } } }, - .amoxorw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } }, - .amoandw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } }, - .amoorw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } }, - .amominw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10000 } } }, - .amomaxw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10100 } } }, - .amominuw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } }, - .amomaxuw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } }, - - - .amoaddd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } }, - .amoswapd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } }, - .lrd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00010 } } }, - .scd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00011 } } }, - .amoxord => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } }, - .amoandd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } }, - .amoord => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } }, - .amomind => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10000 } } }, - .amomaxd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10100 } } }, - .amominud => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11000 } } }, - .amomaxud => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } }, - - // OP_V - .vsetivli => .{ .opcode = .OP_V, .format = .I, .data = .{ .f = .{ .funct3 = 0b111 } } }, - .vsetvli => .{ .opcode = .OP_V, .format = .I, .data = .{ .f = .{ .funct3 = 0b111 } } }, - .vaddvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000000, .funct3 = .OPIVV } } }, - .vsubvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000010, .funct3 = .OPIVV } } }, - .vmulvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b100101, .funct3 = .OPIVV } } }, - - .vfaddvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000000, .funct3 = .OPFVV } } }, - .vfsubvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000010, .funct3 = .OPFVV } } }, - .vfmulvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b100100, .funct3 = .OPFVV } } }, - - .vadcvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b010000, .funct3 = .OPMVV } } }, - .vmvvx => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b010111, .funct3 = .OPIVX } } }, - - .vslidedownvx => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b001111, .funct3 = .OPIVX } } }, - - - .pseudo_prologue, - .pseudo_epilogue, - .pseudo_dbg_prologue_end, - .pseudo_dbg_epilogue_begin, - .pseudo_dbg_line_column, - .pseudo_load_rm, - .pseudo_store_rm, - .pseudo_lea_rm, - .pseudo_j, - .pseudo_dead, - .pseudo_load_symbol, - .pseudo_load_tlv, - .pseudo_mv, - .pseudo_restore_regs, - .pseudo_spill_regs, - .pseudo_compare, - .pseudo_not, - .pseudo_extern_fn_reloc, - .nop, - => std.debug.panic("lir: didn't catch pseudo {s}", .{@tagName(mnem)}), - // zig fmt: on - }; - } -}; - -/// This is the final form of the instruction. Lir is transformed into -/// this, which is then bitcast into a u32. -pub const Instruction = union(Lir.Format) { - R: packed struct(u32) { - opcode: u7, - rd: u5, - funct3: u3, - rs1: u5, - rs2: u5, - funct7: u7, - }, - I: packed struct(u32) { - opcode: u7, - rd: u5, - funct3: u3, - rs1: u5, - imm0_11: u12, - }, - S: packed struct(u32) { - opcode: u7, - imm0_4: u5, - funct3: u3, - rs1: u5, - rs2: u5, - imm5_11: u7, - }, - B: packed struct(u32) { - opcode: u7, - imm11: u1, - imm1_4: u4, - funct3: u3, - rs1: u5, - rs2: u5, - imm5_10: u6, - imm12: u1, - }, - U: packed struct(u32) { - opcode: u7, - rd: u5, - imm12_31: u20, - }, - J: packed struct(u32) { - opcode: u7, - rd: u5, - imm12_19: u8, - imm11: u1, - imm1_10: u10, - imm20: u1, - }, - extra: u32, - - comptime { - for (std.meta.fields(Instruction)) |field| { - assert(@bitSizeOf(field.type) == 32); - } - } - - pub const Operand = union(enum) { - none, - reg: Register, - csr: CSR, - mem: Memory, - imm: Immediate, - barrier: Mir.Barrier, - }; - - pub fn toU32(inst: Instruction) u32 { - return switch (inst) { - inline else => |v| @bitCast(v), - }; - } - - pub fn encode(inst: Instruction, writer: *std.Io.Writer) !void { - try writer.writeInt(u32, inst.toU32(), .little); - } - - pub fn fromLir(lir: Lir, ops: []const Operand) Instruction { - const opcode: u7 = @intFromEnum(lir.opcode); - - switch (lir.format) { - .R => { - return .{ - .R = switch (lir.data) { - .ff => |ff| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .rs2 = ops[2].reg.encodeId(), - - .opcode = opcode, - .funct3 = ff.funct3, - .funct7 = ff.funct7, - }, - .fmt => |fmt| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .rs2 = ops[2].reg.encodeId(), - - .opcode = opcode, - .funct3 = fmt.rm, - .funct7 = (@as(u7, fmt.funct5) << 2) | @intFromEnum(fmt.fmt), - }, - .fcvt => |fcvt| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .rs2 = @intFromEnum(fcvt.width), - - .opcode = opcode, - .funct3 = fcvt.rm, - .funct7 = (@as(u7, fcvt.funct5) << 2) | @intFromEnum(fcvt.fmt), - }, - .vecls => |vec| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - - .rs2 = @intFromEnum(vec.umop), - - .opcode = opcode, - .funct3 = @intFromEnum(vec.width), - .funct7 = (@as(u7, vec.nf) << 4) | (@as(u7, @intFromBool(vec.mew)) << 3) | (@as(u7, @intFromEnum(vec.mop)) << 1) | @intFromBool(vec.vm), - }, - .vecmath => |vec| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .rs2 = ops[2].reg.encodeId(), - - .opcode = opcode, - .funct3 = @intFromEnum(vec.funct3), - .funct7 = (@as(u7, vec.funct6) << 1) | @intFromBool(vec.vm), - }, - .amo => |amo| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .rs2 = ops[2].reg.encodeId(), - - .opcode = opcode, - .funct3 = @intFromEnum(amo.width), - .funct7 = @as(u7, amo.funct5) << 2 | - @as(u7, @intFromBool(ops[3].barrier == .rl)) << 1 | - @as(u7, @intFromBool(ops[4].barrier == .aq)), - }, - else => unreachable, - }, - }; - }, - .S => { - assert(ops.len == 3); - const umm = ops[2].imm.asBits(u12); - return .{ - .S = .{ - .imm0_4 = @truncate(umm), - .rs1 = ops[0].reg.encodeId(), - .rs2 = ops[1].reg.encodeId(), - .imm5_11 = @truncate(umm >> 5), - - .opcode = opcode, - .funct3 = lir.data.f.funct3, - }, - }; - }, - .I => { - return .{ - .I = switch (lir.data) { - .f => |f| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .imm0_11 = ops[2].imm.asBits(u12), - - .opcode = opcode, - .funct3 = f.funct3, - }, - .sh => |sh| .{ - .rd = ops[0].reg.encodeId(), - .rs1 = ops[1].reg.encodeId(), - .imm0_11 = (@as(u12, sh.typ) << 6) | - if (sh.has_5) ops[2].imm.asBits(u6) else (@as(u6, 0) | ops[2].imm.asBits(u5)), - - .opcode = opcode, - .funct3 = sh.funct3, - }, - .fence => |fence| .{ - .rd = 0, - .rs1 = 0, - .funct3 = 0, - .imm0_11 = (@as(u12, @intFromEnum(fence.fm)) << 8) | - (@as(u12, @intFromEnum(ops[1].barrier)) << 4) | - @as(u12, @intFromEnum(ops[0].barrier)), - .opcode = opcode, - }, - else => unreachable, - }, - }; - }, - .U => { - assert(ops.len == 2); - return .{ - .U = .{ - .rd = ops[0].reg.encodeId(), - .imm12_31 = ops[1].imm.asBits(u20), - - .opcode = opcode, - }, - }; - }, - .J => { - assert(ops.len == 2); - - const umm = ops[1].imm.asBits(u21); - // the RISC-V spec says the target index of a jump - // must be a multiple of 2 - assert(umm % 2 == 0); - - return .{ - .J = .{ - .rd = ops[0].reg.encodeId(), - .imm1_10 = @truncate(umm >> 1), - .imm11 = @truncate(umm >> 11), - .imm12_19 = @truncate(umm >> 12), - .imm20 = @truncate(umm >> 20), - - .opcode = opcode, - }, - }; - }, - .B => { - assert(ops.len == 3); - - const umm = ops[2].imm.asBits(u13); - // the RISC-V spec says the target index of a branch - // must be a multiple of 2 - assert(umm % 2 == 0); - - return .{ - .B = .{ - .rs1 = ops[0].reg.encodeId(), - .rs2 = ops[1].reg.encodeId(), - .imm1_4 = @truncate(umm >> 1), - .imm5_10 = @truncate(umm >> 5), - .imm11 = @truncate(umm >> 11), - .imm12 = @truncate(umm >> 12), - - .opcode = opcode, - .funct3 = lir.data.f.funct3, - }, - }; - }, - .extra => { - assert(ops.len == 0); - - return .{ - .I = .{ - .rd = Register.zero.encodeId(), - .rs1 = Register.zero.encodeId(), - .imm0_11 = switch (lir.data.extra) { - .ecall => 0x000, - .ebreak => 0x001, - .unimp => 0x000, - else => unreachable, - }, - - .opcode = opcode, - .funct3 = 0b000, - }, - }; - }, - } - } -}; - -const std = @import("std"); -const assert = std.debug.assert; -const log = std.log.scoped(.format); - -const bits = @import("bits.zig"); -const Mir = @import("Mir.zig"); -const Mnemonic = @import("mnem.zig").Mnemonic; -const Lower = @import("Lower.zig"); - -const Register = bits.Register; -const CSR = bits.CSR; -const Memory = bits.Memory; -const Immediate = bits.Immediate; diff --git a/src/arch/riscv64/mnem.zig b/src/arch/riscv64/mnem.zig deleted file mode 100644 index cb4734e59cf7769a4f1efefb572d1252e1e1ad30..0000000000000000000000000000000000000000 --- a/src/arch/riscv64/mnem.zig +++ /dev/null @@ -1,257 +0,0 @@ -pub const Mnemonic = enum(u16) { - // Arithmetics - addi, - add, - addw, - - sub, - subw, - - // Bits - xori, - xor, - @"or", - - @"and", - andi, - - slt, - sltu, - sltiu, - - slli, - srli, - srai, - - slliw, - srliw, - sraiw, - - sll, - srl, - sra, - - sllw, - srlw, - sraw, - - // Control Flow - jalr, - jal, - - beq, - bne, - - // Memory - lui, - auipc, - - ld, - lw, - lh, - lb, - lbu, - lhu, - lwu, - - sd, - sw, - sh, - sb, - - // System - ebreak, - ecall, - unimp, - nop, - - // M extension - mul, - mulh, - mulhu, - mulhsu, - mulw, - - div, - divu, - divw, - divuw, - - rem, - remu, - remw, - remuw, - - // F extension (32-bit float) - fadds, - fsubs, - fmuls, - fdivs, - - fmins, - fmaxs, - - fsqrts, - - flw, - fsw, - - feqs, - flts, - fles, - - // D extension (64-bit float) - faddd, - fsubd, - fmuld, - fdivd, - - fmind, - fmaxd, - - fsqrtd, - - fld, - fsd, - - feqd, - fltd, - fled, - - fcvtws, - fcvtwus, - fcvtls, - fcvtlus, - - fcvtwd, - fcvtwud, - fcvtld, - fcvtlud, - - fcvtsw, - fcvtswu, - fcvtsl, - fcvtslu, - - fcvtdw, - fcvtdwu, - fcvtdl, - fcvtdlu, - - fsgnjns, - fsgnjnd, - - fsgnjxs, - fsgnjxd, - - // Zicsr Extension Instructions - csrrs, - - // V Extension Instructions - vsetvli, - vsetivli, - vaddvv, - vfaddvv, - vsubvv, - vfsubvv, - vmulvv, - vfmulvv, - vslidedownvx, - - vle8v, - vle16v, - vle32v, - vle64v, - - vse8v, - vse16v, - vse32v, - vse64v, - - vadcvv, - vmvvx, - - // Zbb Extension Instructions - clz, - clzw, - cpop, - cpopw, - - // A Extension Instructions - fence, - fencetso, - - lrw, - scw, - amoswapw, - amoaddw, - amoandw, - amoorw, - amoxorw, - amomaxw, - amominw, - amomaxuw, - amominuw, - - lrd, - scd, - amoswapd, - amoaddd, - amoandd, - amoord, - amoxord, - amomaxd, - amomind, - amomaxud, - amominud, - - // Pseudo-instructions. Used for anything that isn't 1:1 with an - // assembly instruction. - - /// Pseudo-instruction that will generate a backpatched - /// function prologue. - pseudo_prologue, - /// Pseudo-instruction that will generate a backpatched - /// function epilogue - pseudo_epilogue, - - /// Pseudo-instruction: End of prologue - pseudo_dbg_prologue_end, - /// Pseudo-instruction: Beginning of epilogue - pseudo_dbg_epilogue_begin, - /// Pseudo-instruction: Update debug line - pseudo_dbg_line_column, - - /// Pseudo-instruction that loads from memory into a register. - pseudo_load_rm, - /// Pseudo-instruction that stores from a register into memory - pseudo_store_rm, - /// Pseudo-instruction that loads the address of memory into a register. - pseudo_lea_rm, - /// Jumps. Uses `inst` payload. - pseudo_j, - /// Dead inst, ignored by the emitter. - pseudo_dead, - /// Loads the address of a value that hasn't yet been allocated in memory. - pseudo_load_symbol, - /// Loads the address of a TLV. - pseudo_load_tlv, - - /// Moves the value of rs1 to rd. - pseudo_mv, - - pseudo_restore_regs, - pseudo_spill_regs, - - pseudo_compare, - pseudo_not, - pseudo_extern_fn_reloc, -}; - -pub const Pseudo = enum(u8) { - li, - mv, - tail, - beqz, - ret, -}; diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig deleted file mode 100644 index 48d44e39f973c4f1b92604a34ee6258186b425a2..0000000000000000000000000000000000000000 --- a/src/arch/sparc64/CodeGen.zig +++ /dev/null @@ -1,4827 +0,0 @@ -//! SPARCv9 codegen. -//! This lowers AIR into MIR. -//! For now this only implements medium/low code model with absolute addressing. -//! TODO add support for other code models. -const std = @import("std"); -const assert = std.debug.assert; -const log = std.log.scoped(.codegen); -const math = std.math; -const mem = std.mem; -const Allocator = mem.Allocator; -const builtin = @import("builtin"); -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); -const InternPool = @import("../../InternPool.zig"); -const Value = @import("../../Value.zig"); -const ErrorMsg = Zcu.ErrorMsg; -const codegen = @import("../../codegen.zig"); -const Air = @import("../../Air.zig"); -const Mir = @import("Mir.zig"); -const Emit = @import("Emit.zig"); -const Type = @import("../../Type.zig"); -const CodeGenError = codegen.CodeGenError; -const Endian = std.builtin.Endian; -const Alignment = InternPool.Alignment; - -const build_options = @import("build_options"); - -const bits = @import("bits.zig"); -const abi = @import("abi.zig"); -const errUnionPayloadOffset = codegen.errUnionPayloadOffset; -const errUnionErrorOffset = codegen.errUnionErrorOffset; -const Instruction = bits.Instruction; -const ASI = Instruction.ASI; -const ShiftWidth = Instruction.ShiftWidth; -const RegisterManager = abi.RegisterManager; -const RegisterLock = RegisterManager.RegisterLock; -const Register = bits.Register; -const gp = abi.RegisterClass.gp; - -const Self = @This(); - -const InnerError = CodeGenError || error{OutOfRegisters}; - -pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { - return null; -} - -const RegisterView = enum(u1) { - caller, - callee, -}; - -gpa: Allocator, -pt: Zcu.PerThread, -air: Air, -liveness: Air.Liveness, -bin_file: *link.File, -target: *const std.Target, -func_index: InternPool.Index, -err_msg: ?*ErrorMsg, -args: []MCValue, -ret_mcv: MCValue, -fn_type: Type, -arg_index: usize, -src_loc: Zcu.LazySrcLoc, -stack_align: Alignment, - -/// MIR Instructions -mir_instructions: std.MultiArrayList(Mir.Inst) = .{}, -/// MIR extra data -mir_extra: std.ArrayListUnmanaged(u32) = .empty, - -/// Byte offset within the source file of the ending curly. -end_di_line: u32, -end_di_column: u32, - -/// The value is an offset into the `Function` `code` from the beginning. -/// To perform the reloc, write 32-bit signed little-endian integer -/// which is a relative jump, based on the address following the reloc. -exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, - -reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, - -/// Whenever there is a runtime branch, we push a Branch onto this stack, -/// and pop it off when the runtime branch joins. This provides an "overlay" -/// of the table of mappings from instructions to `MCValue` from within the branch. -/// This way we can modify the `MCValue` for an instruction in different ways -/// within different branches. Special consideration is needed when a branch -/// joins with its parent, to make sure all instructions have the same MCValue -/// across each runtime branch upon joining. -branch_stack: *std.array_list.Managed(Branch), - -// Key is the block instruction -blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, - -register_manager: RegisterManager = .{}, - -/// Maps offset to what is stored there. -stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .empty, - -/// Tracks the current instruction allocated to the condition flags -condition_flags_inst: ?Air.Inst.Index = null, - -/// Tracks the current instruction allocated to the condition register -condition_register_inst: ?Air.Inst.Index = null, - -/// Offset from the stack base, representing the end of the stack frame. -max_end_stack: u32 = 0, -/// Represents the current end stack offset. If there is no existing slot -/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`. -next_stack_offset: u32 = 0, - -/// Debug field, used to find bugs in the compiler. -air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init, - -const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; - -const MCValue = union(enum) { - /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. - /// TODO Look into deleting this tag and using `dead` instead, since every use - /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. - none, - /// Control flow will not allow this value to be observed. - unreach, - /// No more references to this value remain. - dead, - /// The value is undefined. - undef, - /// A pointer-sized integer that fits in a register. - /// If the type is a pointer, this is the pointer address in virtual address space. - immediate: u64, - /// The value is in a target-specific register. - register: Register, - /// The value is a tuple { wrapped, overflow } where - /// wrapped is stored in the register and the overflow bit is - /// stored in the C (signed) or V (unsigned) flag of the CCR. - /// - /// This MCValue is only generated by a add_with_overflow or - /// sub_with_overflow instruction operating on 32- or 64-bit values. - register_with_overflow: struct { - reg: Register, - flag: struct { cond: Instruction.ICondition, ccr: Instruction.CCR }, - }, - /// The value is in memory at a hard-coded address. - /// If the type is a pointer, it means the pointer address is at this memory location. - memory: u64, - /// The value is one of the stack variables. - /// If the type is a pointer, it means the pointer address is in the stack at this offset. - /// Note that this stores the plain value (i.e without the effects of the stack bias). - /// Always convert this value into machine offsets with realStackOffset() before - /// lowering into asm! - stack_offset: u32, - /// The value is a pointer to one of the stack variables (payload is stack offset). - ptr_stack_offset: u32, - /// The value is in the specified CCR. The value is 1 (if - /// the type is u1) or true (if the type in bool) iff the - /// specified condition is true. - condition_flags: struct { - cond: Instruction.Condition, - ccr: Instruction.CCR, - }, - /// The value is in the specified Register. The value is 1 (if - /// the type is u1) or true (if the type in bool) iff the - /// specified condition is true. - condition_register: struct { - cond: Instruction.RCondition, - reg: Register, - }, - - fn isMemory(mcv: MCValue) bool { - return switch (mcv) { - .memory, .stack_offset => true, - else => false, - }; - } - - fn isImmediate(mcv: MCValue) bool { - return switch (mcv) { - .immediate => true, - else => false, - }; - } - - fn isMutable(mcv: MCValue) bool { - return switch (mcv) { - .none => unreachable, - .unreach => unreachable, - .dead => unreachable, - - .immediate, - .memory, - .condition_flags, - .condition_register, - .ptr_stack_offset, - .undef, - => false, - - .register, - .stack_offset, - => true, - }; - } -}; - -const Branch = struct { - inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty, - - fn deinit(self: *Branch, gpa: Allocator) void { - self.inst_table.deinit(gpa); - self.* = undefined; - } -}; - -const StackAllocation = struct { - inst: Air.Inst.Index, - /// TODO do we need size? should be determined by inst.ty.abiSize() - size: u32, -}; - -const BlockData = struct { - relocs: std.ArrayListUnmanaged(Mir.Inst.Index), - /// The first break instruction encounters `null` here and chooses a - /// machine code value for the block result, populating this field. - /// Following break instructions encounter that value and use it for - /// the location to store their block results. - mcv: MCValue, -}; - -const CallMCValues = struct { - args: []MCValue, - return_value: MCValue, - stack_byte_count: u32, - stack_align: Alignment, - - fn deinit(self: *CallMCValues, func: *Self) void { - func.gpa.free(self.args); - self.* = undefined; - } -}; - -const BigTomb = struct { - function: *Self, - inst: Air.Inst.Index, - lbt: Air.Liveness.BigTomb, - - fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { - const dies = bt.lbt.feed(); - const op_index = op_ref.toIndex() orelse return; - if (!dies) return; - bt.function.processDeath(op_index); - } - - fn finishAir(bt: *BigTomb, result: MCValue) void { - const is_used = !bt.function.liveness.isUnused(bt.inst); - if (is_used) { - log.debug("%{d} => {}", .{ bt.inst, result }); - const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1]; - branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result); - } - bt.function.finishAirBookkeeping(); - } -}; - -pub fn generate( - lf: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - air: *const Air, - liveness: *const ?Air.Liveness, -) CodeGenError!Mir { - const zcu = pt.zcu; - const gpa = zcu.gpa; - const func = zcu.funcInfo(func_index); - const func_ty = Type.fromInterned(func.ty); - const file_scope = zcu.navFileScope(func.owner_nav); - const target = &file_scope.mod.?.resolved_target.result; - - var branch_stack = std.array_list.Managed(Branch).init(gpa); - defer { - assert(branch_stack.items.len == 1); - branch_stack.items[0].deinit(gpa); - branch_stack.deinit(); - } - try branch_stack.append(.{}); - - var function: Self = .{ - .gpa = gpa, - .pt = pt, - .air = air.*, - .liveness = liveness.*.?, - .target = target, - .bin_file = lf, - .func_index = func_index, - .err_msg = null, - .args = undefined, // populated after `resolveCallingConventionValues` - .ret_mcv = undefined, // populated after `resolveCallingConventionValues` - .fn_type = func_ty, - .arg_index = 0, - .branch_stack = &branch_stack, - .src_loc = src_loc, - .stack_align = undefined, - .end_di_line = func.rbrace_line, - .end_di_column = func.rbrace_column, - }; - defer function.stack.deinit(gpa); - defer function.blocks.deinit(gpa); - defer function.exitlude_jump_relocs.deinit(gpa); - - var call_info = function.resolveCallingConventionValues(func_ty, .callee) catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - else => |e| return e, - }; - defer call_info.deinit(&function); - - function.args = call_info.args; - function.ret_mcv = call_info.return_value; - function.stack_align = call_info.stack_align; - function.max_end_stack = call_info.stack_byte_count; - - function.gen() catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), - else => |e| return e, - }; - - var mir: Mir = .{ - .instructions = function.mir_instructions.toOwnedSlice(), - .extra = &.{}, // fallible, so populated after errdefer - }; - errdefer mir.deinit(gpa); - mir.extra = try function.mir_extra.toOwnedSlice(gpa); - return mir; -} - -fn gen(self: *Self) !void { - const pt = self.pt; - const zcu = pt.zcu; - const cc = self.fn_type.fnCallingConvention(zcu); - if (cc != .naked) { - // TODO Finish function prologue and epilogue for sparc64. - - // save %sp, stack_reserved_area, %sp - const save_inst = try self.addInst(.{ - .tag = .save, - .data = .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = .sp, - .rs1 = .sp, - .rs2_or_imm = .{ .imm = -abi.stack_reserved_area }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .dbg_prologue_end, - .data = .{ .nop = {} }, - }); - - try self.genBody(self.air.getMainBody()); - - _ = try self.addInst(.{ - .tag = .dbg_epilogue_begin, - .data = .{ .nop = {} }, - }); - - // exitlude jumps - if (self.exitlude_jump_relocs.items.len > 0 and - self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 3) - { - // If the last Mir instruction (apart from the - // dbg_epilogue_begin) is the last exitlude jump - // relocation (which would just jump two instructions - // further), it can be safely removed - const index = self.exitlude_jump_relocs.pop().?; - - // First, remove the delay slot, then remove - // the branch instruction itself. - self.mir_instructions.orderedRemove(index + 1); - self.mir_instructions.orderedRemove(index); - } - - for (self.exitlude_jump_relocs.items) |jmp_reloc| { - self.mir_instructions.set(jmp_reloc, .{ - .tag = .bpcc, - .data = .{ - .branch_predict_int = .{ - .ccr = .xcc, - .cond = .al, - .inst = @as(u32, @intCast(self.mir_instructions.len)), - }, - }, - }); - } - - // Backpatch stack offset - const total_stack_size = self.max_end_stack + abi.stack_reserved_area; - const stack_size = self.stack_align.forward(total_stack_size); - if (math.cast(i13, stack_size)) |size| { - self.mir_instructions.set(save_inst, .{ - .tag = .save, - .data = .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = .sp, - .rs1 = .sp, - .rs2_or_imm = .{ .imm = -size }, - }, - }, - }); - } else { - // TODO for large stacks, replace the prologue with: - // setx stack_size, %g1 - // save %sp, %g1, %sp - return self.fail("TODO SPARCv9: allow larger stacks", .{}); - } - - // return %i7 + 8 - _ = try self.addInst(.{ - .tag = .@"return", - .data = .{ - .arithmetic_2op = .{ - .is_imm = true, - .rs1 = .i7, - .rs2_or_imm = .{ .imm = 8 }, - }, - }, - }); - - // Branches in SPARC have a delay slot, that is, the instruction - // following it will unconditionally be executed. - // See: Section 3.2.3 Control Transfer in SPARCv9 manual. - // See also: https://arcb.csc.ncsu.edu/~mueller/codeopt/codeopt00/notes/delaybra.html - // TODO Find a way to fill this delay slot - // nop - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); - } else { - _ = try self.addInst(.{ - .tag = .dbg_prologue_end, - .data = .{ .nop = {} }, - }); - - try self.genBody(self.air.getMainBody()); - - _ = try self.addInst(.{ - .tag = .dbg_epilogue_begin, - .data = .{ .nop = {} }, - }); - } - - // Drop them off at the rbrace. - _ = try self.addInst(.{ - .tag = .dbg_line, - .data = .{ .dbg_line_column = .{ - .line = self.end_di_line, - .column = self.end_di_column, - } }, - }); -} - -fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const air_tags = self.air.instructions.items(.tag); - - for (body) |inst| { - // TODO: remove now-redundant isUnused calls from AIR handler functions - if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) - continue; - - const old_air_bookkeeping = self.air_bookkeeping; - try self.ensureProcessDeathCapacity(Air.Liveness.bpi); - - self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); - switch (air_tags[@intFromEnum(inst)]) { - // zig fmt: off - .ptr_add => try self.airPtrArithmetic(inst, .ptr_add), - .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub), - - .add => try self.airBinOp(inst, .add), - .add_wrap => try self.airBinOp(inst, .add_wrap), - .sub => try self.airBinOp(inst, .sub), - .sub_wrap => try self.airBinOp(inst, .sub_wrap), - .mul => try self.airBinOp(inst, .mul), - .mul_wrap => try self.airBinOp(inst, .mul_wrap), - .shl => try self.airBinOp(inst, .shl), - .shl_exact => try self.airBinOp(inst, .shl_exact), - .shr => try self.airBinOp(inst, .shr), - .shr_exact => try self.airBinOp(inst, .shr_exact), - .bool_and => try self.airBinOp(inst, .bool_and), - .bool_or => try self.airBinOp(inst, .bool_or), - .bit_and => try self.airBinOp(inst, .bit_and), - .bit_or => try self.airBinOp(inst, .bit_or), - .xor => try self.airBinOp(inst, .xor), - - .add_sat => try self.airAddSat(inst), - .sub_sat => try self.airSubSat(inst), - .mul_sat => try self.airMulSat(inst), - .shl_sat => try self.airShlSat(inst), - .min, .max => try self.airMinMax(inst), - .rem => try self.airRem(inst), - .mod => try self.airMod(inst), - .slice => try self.airSlice(inst), - - .sqrt, - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .abs, - .floor, - .ceil, - .round, - .trunc_float, - .neg, - => try self.airUnaryMath(inst), - - .add_with_overflow => try self.airAddSubWithOverflow(inst), - .sub_with_overflow => try self.airAddSubWithOverflow(inst), - .mul_with_overflow => try self.airMulWithOverflow(inst), - .shl_with_overflow => try self.airShlWithOverflow(inst), - - .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst), - - .cmp_lt => try self.airCmp(inst, .lt), - .cmp_lte => try self.airCmp(inst, .lte), - .cmp_eq => try self.airCmp(inst, .eq), - .cmp_gte => try self.airCmp(inst, .gte), - .cmp_gt => try self.airCmp(inst, .gt), - .cmp_neq => try self.airCmp(inst, .neq), - .cmp_vector => @panic("TODO try self.airCmpVector(inst)"), - .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), - - .alloc => try self.airAlloc(inst), - .ret_ptr => try self.airRetPtr(inst), - .arg => try self.airArg(inst), - .assembly => try self.airAsm(inst), - .bitcast => try self.airBitCast(inst), - .block => try self.airBlock(inst), - .br => try self.airBr(inst), - .repeat => return self.fail("TODO implement `repeat`", .{}), - .switch_dispatch => return self.fail("TODO implement `switch_dispatch`", .{}), - .trap => try self.airTrap(), - .breakpoint => try self.airBreakpoint(), - .ret_addr => @panic("TODO try self.airRetAddr(inst)"), - .frame_addr => @panic("TODO try self.airFrameAddress(inst)"), - .cond_br => try self.airCondBr(inst), - .fptrunc => @panic("TODO try self.airFptrunc(inst)"), - .fpext => @panic("TODO try self.airFpext(inst)"), - .intcast => try self.airIntCast(inst), - .trunc => try self.airTrunc(inst), - .is_non_null => try self.airIsNonNull(inst), - .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"), - .is_null => try self.airIsNull(inst), - .is_null_ptr => @panic("TODO try self.airIsNullPtr(inst)"), - .is_non_err => try self.airIsNonErr(inst), - .is_non_err_ptr => @panic("TODO try self.airIsNonErrPtr(inst)"), - .is_err => try self.airIsErr(inst), - .is_err_ptr => @panic("TODO try self.airIsErrPtr(inst)"), - .load => try self.airLoad(inst), - .loop => try self.airLoop(inst), - .not => try self.airNot(inst), - .ret => try self.airRet(inst), - .ret_safe => try self.airRet(inst), // TODO - .ret_load => try self.airRetLoad(inst), - .store => try self.airStore(inst, false), - .store_safe => try self.airStore(inst, true), - .struct_field_ptr=> try self.airStructFieldPtr(inst), - .struct_field_val=> try self.airStructFieldVal(inst), - .array_to_slice => try self.airArrayToSlice(inst), - .float_from_int => try self.airFloatFromInt(inst), - .int_from_float => try self.airIntFromFloat(inst), - .cmpxchg_strong, - .cmpxchg_weak, - => try self.airCmpxchg(inst), - .atomic_rmw => try self.airAtomicRmw(inst), - .atomic_load => try self.airAtomicLoad(inst), - .memcpy => @panic("TODO try self.airMemcpy(inst)"), - .memmove => @panic("TODO try self.airMemmove(inst)"), - .memset => try self.airMemset(inst, false), - .memset_safe => try self.airMemset(inst, true), - .set_union_tag => try self.airSetUnionTag(inst), - .get_union_tag => try self.airGetUnionTag(inst), - .clz => try self.airClz(inst), - .ctz => try self.airCtz(inst), - .popcount => try self.airPopcount(inst), - .byte_swap => try self.airByteSwap(inst), - .bit_reverse => try self.airBitReverse(inst), - .tag_name => try self.airTagName(inst), - .error_name => try self.airErrorName(inst), - .splat => try self.airSplat(inst), - .select => @panic("TODO try self.airSelect(inst)"), - .shuffle_one => @panic("TODO try self.airShuffleOne(inst)"), - .shuffle_two => @panic("TODO try self.airShuffleTwo(inst)"), - .reduce => @panic("TODO try self.airReduce(inst)"), - .aggregate_init => try self.airAggregateInit(inst), - .union_init => try self.airUnionInit(inst), - .prefetch => try self.airPrefetch(inst), - .mul_add => @panic("TODO try self.airMulAdd(inst)"), - .addrspace_cast => @panic("TODO try self.airAddrSpaceCast(int)"), - - .@"try" => try self.airTry(inst), - .try_cold => try self.airTry(inst), - .try_ptr => @panic("TODO try self.airTryPtr(inst)"), - .try_ptr_cold => @panic("TODO try self.airTryPtrCold(inst)"), - - .dbg_stmt => try self.airDbgStmt(inst), - .dbg_empty_stmt => self.finishAirBookkeeping(), - .dbg_inline_block => try self.airDbgInlineBlock(inst), - .dbg_var_ptr, - .dbg_var_val, - .dbg_arg_inline, - => try self.airDbgVar(inst), - - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), - - .atomic_store_unordered => @panic("TODO try self.airAtomicStore(inst, .unordered)"), - .atomic_store_monotonic => @panic("TODO try self.airAtomicStore(inst, .monotonic)"), - .atomic_store_release => @panic("TODO try self.airAtomicStore(inst, .release)"), - .atomic_store_seq_cst => @panic("TODO try self.airAtomicStore(inst, .seq_cst)"), - - .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), - .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), - .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), - .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), - - .field_parent_ptr => @panic("TODO try self.airFieldParentPtr(inst)"), - - .switch_br => try self.airSwitch(inst), - .loop_switch_br => return self.fail("TODO implement `loop_switch_br`", .{}), - .slice_ptr => try self.airSlicePtr(inst), - .slice_len => try self.airSliceLen(inst), - - .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst), - .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst), - - .array_elem_val => try self.airArrayElemVal(inst), - .slice_elem_val => try self.airSliceElemVal(inst), - .slice_elem_ptr => @panic("TODO try self.airSliceElemPtr(inst)"), - .ptr_elem_val => try self.airPtrElemVal(inst), - .ptr_elem_ptr => try self.airPtrElemPtr(inst), - - .inferred_alloc, .inferred_alloc_comptime => unreachable, - .unreach => self.finishAirBookkeeping(), - - .optional_payload => try self.airOptionalPayload(inst), - .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), - .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), - .unwrap_errunion_err => try self.airUnwrapErrErr(inst), - .unwrap_errunion_payload => try self.airUnwrapErrPayload(inst), - .unwrap_errunion_err_ptr => @panic("TODO try self.airUnwrapErrErrPtr(inst)"), - .unwrap_errunion_payload_ptr=> @panic("TODO try self.airUnwrapErrPayloadPtr(inst)"), - .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), - .err_return_trace => @panic("TODO try self.airErrReturnTrace(inst)"), - .set_err_return_trace => @panic("TODO try self.airSetErrReturnTrace(inst)"), - .save_err_return_trace_index=> @panic("TODO try self.airSaveErrReturnTraceIndex(inst)"), - - .wrap_optional => try self.airWrapOptional(inst), - .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst), - .wrap_errunion_err => try self.airWrapErrUnionErr(inst), - - .add_optimized, - .sub_optimized, - .mul_optimized, - .div_float_optimized, - .div_trunc_optimized, - .div_floor_optimized, - .div_exact_optimized, - .rem_optimized, - .mod_optimized, - .neg_optimized, - .cmp_lt_optimized, - .cmp_lte_optimized, - .cmp_eq_optimized, - .cmp_gte_optimized, - .cmp_gt_optimized, - .cmp_neq_optimized, - .cmp_vector_optimized, - .reduce_optimized, - .int_from_float_optimized, - => @panic("TODO implement optimized float mode"), - - .add_safe, - .sub_safe, - .mul_safe, - .intcast_safe, - .int_from_float_safe, - .int_from_float_optimized_safe, - => @panic("TODO implement safety_checked_instructions"), - - .is_named_enum_value => @panic("TODO implement is_named_enum_value"), - .error_set_has_value => @panic("TODO implement error_set_has_value"), - .vector_store_elem => @panic("TODO implement vector_store_elem"), - .runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"), - - .c_va_arg => return self.fail("TODO implement c_va_arg", .{}), - .c_va_copy => return self.fail("TODO implement c_va_copy", .{}), - .c_va_end => return self.fail("TODO implement c_va_end", .{}), - .c_va_start => return self.fail("TODO implement c_va_start", .{}), - - .wasm_memory_size => unreachable, - .wasm_memory_grow => unreachable, - - .work_item_id => unreachable, - .work_group_size => unreachable, - .work_group_id => unreachable, - // zig fmt: on - } - - assert(!self.register_manager.lockedRegsExist()); - - if (std.debug.runtime_safety) { - if (self.air_bookkeeping < old_air_bookkeeping + 1) { - std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{t}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] }); - } - } - } -} - -fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { - const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - const pt = self.pt; - const zcu = pt.zcu; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const lhs = try self.resolveInst(extra.lhs); - const rhs = try self.resolveInst(extra.rhs); - const lhs_ty = self.typeOf(extra.lhs); - const rhs_ty = self.typeOf(extra.rhs); - - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}), - .int => { - assert(lhs_ty.eql(rhs_ty, zcu)); - const int_info = lhs_ty.intInfo(zcu); - switch (int_info.bits) { - 32, 64 => { - // Only say yes if the operation is - // commutative, i.e. we can swap both of the - // operands - const lhs_immediate_ok = switch (tag) { - .add_with_overflow => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), - .sub_with_overflow => false, - else => unreachable, - }; - const rhs_immediate_ok = switch (tag) { - .add_with_overflow, - .sub_with_overflow, - => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), - else => unreachable, - }; - - const mir_tag: Mir.Inst.Tag = switch (tag) { - .add_with_overflow => .addcc, - .sub_with_overflow => .subcc, - else => unreachable, - }; - - try self.spillConditionFlagsIfOccupied(); - - const dest = blk: { - if (rhs_immediate_ok) { - break :blk try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, null); - } else if (lhs_immediate_ok) { - // swap lhs and rhs - break :blk try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, null); - } else { - break :blk try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, null); - } - }; - - const cond = switch (int_info.signedness) { - .unsigned => switch (tag) { - .add_with_overflow => Instruction.ICondition.cs, - .sub_with_overflow => Instruction.ICondition.cc, - else => unreachable, - }, - .signed => Instruction.ICondition.vs, - }; - - const ccr = switch (int_info.bits) { - 32 => Instruction.CCR.icc, - 64 => Instruction.CCR.xcc, - else => unreachable, - }; - - break :result MCValue{ .register_with_overflow = .{ - .reg = dest.register, - .flag = .{ .cond = cond, .ccr = ccr }, - } }; - }, - else => return self.fail("TODO overflow operations on other integer sizes", .{}), - } - }, - else => unreachable, - } - }; - return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const vector_ty = self.typeOfIndex(inst); - const len = vector_ty.vectorLen(zcu); - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); - const result: MCValue = res: { - if (self.liveness.isUnused(inst)) break :res MCValue.dead; - return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch}); - }; - - if (elements.len <= Air.Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); - @memcpy(buf[0..elements.len], elements); - return self.finishAir(inst, result, buf); - } - var bt = try self.iterateBigTomb(inst, elements.len); - for (elements) |elem| { - bt.feed(elem); - } - return bt.finishAir(result); -} - -fn airAlloc(self: *Self, inst: Air.Inst.Index) !void { - const stack_offset = try self.allocMemPtr(inst); - return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none }); -} - -fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const ptr_ty = self.typeOf(ty_op.operand); - const ptr = try self.resolveInst(ty_op.operand); - const array_ty = ptr_ty.childType(zcu); - const array_len: u32 = @intCast(array_ty.arrayLen(zcu)); - const ptr_bytes = 8; - const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8"); - try self.genSetStack(ptr_ty, stack_offset, ptr); - try self.genSetStack(Type.usize, stack_offset - ptr_bytes, .{ .immediate = array_len }); - break :result MCValue{ .stack_offset = stack_offset }; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airAsm(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Asm, ty_pl.payload); - const is_volatile = extra.data.flags.is_volatile; - const outputs_len = extra.data.flags.outputs_len; - var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + outputs_len]); - extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.inputs_len]); - extra_i += inputs.len; - - const dead = !is_volatile and self.liveness.isUnused(inst); - const result: MCValue = if (dead) .dead else result: { - if (outputs.len > 1) { - return self.fail("TODO implement codegen for asm with more than 1 output", .{}); - } - - const output_constraint: ?[]const u8 = for (outputs) |output| { - if (output != .none) { - return self.fail("TODO implement codegen for non-expr asm", .{}); - } - const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - break constraint; - } else null; - - for (inputs) |input| { - const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); - const constraint = std.mem.sliceTo(input_bytes, 0); - const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') { - return self.fail("unrecognized asm input constraint: '{s}'", .{constraint}); - } - const reg_name = constraint[1 .. constraint.len - 1]; - const reg = parseRegName(reg_name) orelse - return self.fail("unrecognized register: '{s}'", .{reg_name}); - - const arg_mcv = try self.resolveInst(input); - try self.register_manager.getReg(reg, null); - try self.genSetReg(self.typeOf(input), reg, arg_mcv); - } - - // TODO honor the clobbers - _ = extra.data.clobbers; - - const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; - - if (mem.eql(u8, asm_source, "ta 0x6d")) { - _ = try self.addInst(.{ - .tag = .tcc, - .data = .{ - .trap = .{ - .is_imm = true, - .cond = .al, - .rs2_or_imm = .{ .imm = 0x6d }, - }, - }, - }); - } else { - return self.fail("TODO implement a full SPARCv9 assembly parsing", .{}); - } - - if (output_constraint) |output| { - if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { - return self.fail("unrecognized asm output constraint: '{s}'", .{output}); - } - const reg_name = output[2 .. output.len - 1]; - const reg = parseRegName(reg_name) orelse - return self.fail("unrecognized register: '{s}'", .{reg_name}); - break :result MCValue{ .register = reg }; - } else { - break :result MCValue{ .none = {} }; - } - }; - - simple: { - var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); - var buf_index: usize = 0; - for (outputs) |output| { - if (output == .none) continue; - - if (buf_index >= buf.len) break :simple; - buf[buf_index] = output; - buf_index += 1; - } - if (buf_index + inputs.len > buf.len) break :simple; - @memcpy(buf[buf_index..][0..inputs.len], inputs); - return self.finishAir(inst, result, buf); - } - - var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len); - for (outputs) |output| { - if (output == .none) continue; - - bt.feed(output); - } - for (inputs) |input| { - bt.feed(input); - } - return bt.finishAir(result); -} - -fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const arg_index = self.arg_index; - self.arg_index += 1; - - const ty = self.typeOfIndex(inst); - const mcv: MCValue = blk: { - switch (self.args[arg_index]) { - .stack_offset => |off| { - const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse { - return self.fail("type '{f}' too big to fit into stack frame", .{ty.fmt(pt)}); - }; - const offset = off + abi_size; - break :blk .{ .stack_offset = offset }; - }, - else => |mcv| break :blk mcv, - } - }; - - const func_zir = zcu.funcInfo(self.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?; - const file = zcu.fileByIndex(func_zir.file); - if (!file.mod.?.strip) { - const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg; - const zir = &file.zir.?; - const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); - - self.genArgDbgInfo(name, ty, mcv) catch |err| - return self.fail("failed to generate debug info for parameter: {s}", .{@errorName(err)}); - } - - if (self.liveness.isUnused(inst)) - return self.finishAirBookkeeping(); - - switch (mcv) { - .register => |reg| { - self.register_manager.getRegAssumeFree(reg, inst); - }, - else => {}, - } - - return self.finishAir(inst, mcv, .{ .none, .none, .none }); -} - -fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void { - _ = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; - - return self.fail("TODO implement airAtomicLoad for {}", .{ - self.target.cpu.arch, - }); -} - -fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { - _ = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - - return self.fail("TODO implement airAtomicRmw for {}", .{ - self.target.cpu.arch, - }); -} - -fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.binOp(tag, lhs, rhs, lhs_ty, rhs_ty, BinOpMetadata{ - .lhs = bin_op.lhs, - .rhs = bin_op.rhs, - .inst = inst, - }); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.binOp(tag, lhs, rhs, lhs_ty, rhs_ty, BinOpMetadata{ - .lhs = bin_op.lhs, - .rhs = bin_op.rhs, - .inst = inst, - }); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(ty_op.operand); - if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand; - - const operand_lock = switch (operand) { - .register => |reg| self.register_manager.lockReg(reg), - .register_with_overflow => |rwo| self.register_manager.lockReg(rwo.reg), - else => null, - }; - defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); - - const dest = try self.allocRegOrMem(inst, true); - try self.setRegOrMem(self.typeOfIndex(inst), dest, operand); - break :result dest; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airBlock(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Block, ty_pl.payload); - try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); -} - -fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { - try self.blocks.putNoClobber(self.gpa, inst, .{ - // A block is a setup to be able to jump to the end. - .relocs = .{}, - // It also acts as a receptacle for break operands. - // Here we use `MCValue.none` to represent a null value so that the first - // break instruction will choose a MCValue for the block result and overwrite - // this field. Following break instructions will use that MCValue to put their - // block results. - .mcv = MCValue{ .none = {} }, - }); - defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa); - - // TODO emit debug info lexical block - try self.genBody(body); - - // relocations for `bpcc` instructions - const relocs = &self.blocks.getPtr(inst).?.relocs; - if (relocs.items.len > 0 and relocs.items[relocs.items.len - 1] == self.mir_instructions.len - 1) { - // If the last Mir instruction is the last relocation (which - // would just jump two instruction further), it can be safely - // removed - const index = relocs.pop().?; - - // First, remove the delay slot, then remove - // the branch instruction itself. - self.mir_instructions.orderedRemove(index + 1); - self.mir_instructions.orderedRemove(index); - } - for (relocs.items) |reloc| { - try self.performReloc(reloc); - } - - const result = self.blocks.getPtr(inst).?.mcv; - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airBr(self: *Self, inst: Air.Inst.Index) !void { - const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; - try self.br(branch.block_inst, branch.operand); - return self.finishAir(inst, .dead, .{ branch.operand, .none, .none }); -} - -fn airTrap(self: *Self) !void { - // ta 0x05 - _ = try self.addInst(.{ - .tag = .tcc, - .data = .{ - .trap = .{ - .is_imm = true, - .cond = .al, - .rs2_or_imm = .{ .imm = 0x05 }, - }, - }, - }); - return self.finishAirBookkeeping(); -} - -fn airBreakpoint(self: *Self) !void { - // ta 0x01 - _ = try self.addInst(.{ - .tag = .tcc, - .data = .{ - .trap = .{ - .is_imm = true, - .cond = .al, - .rs2_or_imm = .{ .imm = 0x01 }, - }, - }, - }); - return self.finishAirBookkeeping(); -} - -fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - // We have hardware byteswapper in SPARCv9, don't let mainstream compilers mislead you. - // That being said, the strategy to lower this is: - // - If src is an immediate, comptime-swap it. - // - If src is in memory then issue an LD*A with #ASI_P_[oppposite-endian] - // - If src is a register then issue an ST*A with #ASI_P_[oppposite-endian] - // to a stack slot, then follow with a normal load from said stack slot. - // This is because on some implementations, ASI-tagged memory operations are non-piplelinable - // and loads tend to have longer latency than stores, so the sequence will minimize stall. - // The result will always be either another immediate or stored in a register. - // TODO: Fold byteswap+store into a single ST*A and load+byteswap into a single LD*A. - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - switch (operand_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO byteswap for vectors", .{}), - .int => { - const int_info = operand_ty.intInfo(zcu); - if (int_info.bits == 8) break :result operand; - - const abi_size = int_info.bits >> 3; - const abi_align = operand_ty.abiAlignment(zcu); - const opposite_endian_asi = switch (self.target.cpu.arch.endian()) { - Endian.big => ASI.asi_primary_little, - Endian.little => ASI.asi_primary, - }; - - switch (operand) { - .immediate => |imm| { - const swapped = switch (int_info.bits) { - 16 => @byteSwap(@as(u16, @intCast(imm))), - 24 => @byteSwap(@as(u24, @intCast(imm))), - 32 => @byteSwap(@as(u32, @intCast(imm))), - 40 => @byteSwap(@as(u40, @intCast(imm))), - 48 => @byteSwap(@as(u48, @intCast(imm))), - 56 => @byteSwap(@as(u56, @intCast(imm))), - 64 => @byteSwap(@as(u64, @intCast(imm))), - else => return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}), - }; - break :result .{ .immediate = swapped }; - }, - .register => |reg| { - if (int_info.bits > 64 or @popCount(int_info.bits) != 1) - return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}); - - const off = try self.allocMem(inst, abi_size, abi_align); - const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) }); - - try self.genStoreASI(reg, .sp, off_reg, abi_size, opposite_endian_asi); - try self.genLoad(reg, .sp, Register, off_reg, abi_size); - break :result .{ .register = reg }; - }, - .memory => { - if (int_info.bits > 64 or @popCount(int_info.bits) != 1) - return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}); - - const addr_reg = try self.copyToTmpRegister(operand_ty, operand); - const dst_reg = try self.register_manager.allocReg(null, gp); - - try self.genLoadASI(dst_reg, addr_reg, .g0, abi_size, opposite_endian_asi); - break :result .{ .register = dst_reg }; - }, - .stack_offset => |off| { - if (int_info.bits > 64 or @popCount(int_info.bits) != 1) - return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}); - - const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) }); - const dst_reg = try self.register_manager.allocReg(null, gp); - - try self.genLoadASI(dst_reg, .sp, off_reg, abi_size, opposite_endian_asi); - break :result .{ .register = dst_reg }; - }, - else => unreachable, - } - }, - else => unreachable, - } - }; - - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void { - if (modifier == .always_tail) return self.fail("TODO implement tail calls for {}", .{self.target.cpu.arch}); - - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const callee = pl_op.operand; - const extra = self.air.extraData(Air.Call, pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end .. extra.end + extra.data.args_len]); - const ty = self.typeOf(callee); - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const fn_ty = switch (ty.zigTypeTag(zcu)) { - .@"fn" => ty, - .pointer => ty.childType(zcu), - else => unreachable, - }; - - var info = try self.resolveCallingConventionValues(fn_ty, .caller); - defer info.deinit(self); - - // CCR is volatile across function calls - // (SCD 2.4.1, page 3P-10) - try self.spillConditionFlagsIfOccupied(); - - // Save caller-saved registers, but crucially *after* we save the - // compare flags as saving compare flags may require a new - // caller-saved register - for (abi.caller_preserved_regs) |reg| { - try self.register_manager.getReg(reg, null); - } - - for (info.args, 0..) |mc_arg, arg_i| { - const arg = args[arg_i]; - const arg_ty = self.typeOf(arg); - const arg_mcv = try self.resolveInst(arg); - - switch (mc_arg) { - .none => continue, - .register => |reg| { - try self.register_manager.getReg(reg, null); - try self.genSetReg(arg_ty, reg, arg_mcv); - }, - .stack_offset => { - return self.fail("TODO implement calling with parameters in memory", .{}); - }, - .ptr_stack_offset => { - return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); - }, - else => unreachable, - } - } - - // Due to incremental compilation, how function calls are generated depends - // on linking. - if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { - .func => { - return self.fail("TODO implement calling functions", .{}); - }, - .@"extern" => { - return self.fail("TODO implement calling extern functions", .{}); - }, - else => { - return self.fail("TODO implement calling bitcasted functions", .{}); - }, - } else { - assert(ty.zigTypeTag(zcu) == .pointer); - const mcv = try self.resolveInst(callee); - try self.genSetReg(ty, .o7, mcv); - - _ = try self.addInst(.{ - .tag = .jmpl, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = .o7, - .rs1 = .o7, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - // TODO Find a way to fill this delay slot - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); - } - - const result = info.return_value; - - if (args.len + 1 <= Air.Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); - buf[0] = callee; - @memcpy(buf[1..][0..args.len], args); - return self.finishAir(inst, result, buf); - } - - var bt = try self.iterateBigTomb(inst, 1 + args.len); - bt.feed(callee); - for (args) |arg| { - bt.feed(arg); - } - return bt.finishAir(result); -} - -fn airClz(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const pt = self.pt; - const zcu = pt.zcu; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const lhs_ty = self.typeOf(bin_op.lhs); - - const int_ty = switch (lhs_ty.zigTypeTag(zcu)) { - .vector => unreachable, // Handled by cmp_vector. - .@"enum" => lhs_ty.intTagType(zcu), - .int => lhs_ty, - .bool => Type.u1, - .pointer => Type.usize, - .error_set => Type.u16, - .optional => blk: { - const payload_ty = lhs_ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :blk Type.u1; - } else if (lhs_ty.isPtrLikeOptional(zcu)) { - break :blk Type.usize; - } else { - return self.fail("TODO SPARCv9 cmp non-pointer optionals", .{}); - } - }, - .float => return self.fail("TODO SPARCv9 cmp floats", .{}), - else => unreachable, - }; - - const int_info = int_ty.intInfo(zcu); - if (int_info.bits <= 64) { - _ = try self.binOp(.cmp_eq, lhs, rhs, int_ty, int_ty, BinOpMetadata{ - .lhs = bin_op.lhs, - .rhs = bin_op.rhs, - .inst = inst, - }); - - try self.spillConditionFlagsIfOccupied(); - self.condition_flags_inst = inst; - - break :result switch (int_info.signedness) { - .signed => MCValue{ .condition_flags = .{ - .cond = .{ .icond = Instruction.ICondition.fromCompareOperatorSigned(op) }, - .ccr = .xcc, - } }, - .unsigned => MCValue{ .condition_flags = .{ - .cond = .{ .icond = Instruction.ICondition.fromCompareOperatorUnsigned(op) }, - .ccr = .xcc, - } }, - }; - } else { - return self.fail("TODO SPARCv9 cmp for ints > 64 bits", .{}); - } - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - _ = operand; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Block, ty_pl.payload); - _ = extra; - - return self.fail("TODO implement airCmpxchg for {}", .{ - self.target.cpu.arch, - }); -} - -fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition = try self.resolveInst(pl_op.operand); - const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); - const liveness_condbr = self.liveness.getCondBr(inst); - - // Here we emit a branch to the false section. - const reloc: Mir.Inst.Index = try self.condBr(condition); - - // If the condition dies here in this condbr instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - if (self.liveness.operandDies(inst, 0)) { - if (pl_op.operand.toIndex()) |op_index| { - self.processDeath(op_index); - } - } - - // Capture the state of register and stack allocation state so that we can revert to it. - const parent_next_stack_offset = self.next_stack_offset; - const parent_free_registers = self.register_manager.free_registers; - var parent_stack = try self.stack.clone(self.gpa); - defer parent_stack.deinit(self.gpa); - const parent_registers = self.register_manager.registers; - const parent_condition_flags_inst = self.condition_flags_inst; - - try self.branch_stack.append(.{}); - errdefer { - _ = self.branch_stack.pop().?; - } - - try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len); - for (liveness_condbr.then_deaths) |operand| { - self.processDeath(operand); - } - try self.genBody(then_body); - - // Revert to the previous register and stack allocation state. - - var saved_then_branch = self.branch_stack.pop().?; - defer saved_then_branch.deinit(self.gpa); - - self.register_manager.registers = parent_registers; - self.condition_flags_inst = parent_condition_flags_inst; - - self.stack.deinit(self.gpa); - self.stack = parent_stack; - parent_stack = .{}; - - self.next_stack_offset = parent_next_stack_offset; - self.register_manager.free_registers = parent_free_registers; - - try self.performReloc(reloc); - const else_branch = self.branch_stack.addOneAssumeCapacity(); - else_branch.* = .{}; - - try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len); - for (liveness_condbr.else_deaths) |operand| { - self.processDeath(operand); - } - try self.genBody(else_body); - - // At this point, each branch will possibly have conflicting values for where - // each instruction is stored. They agree, however, on which instructions are alive/dead. - // We use the first ("then") branch as canonical, and here emit - // instructions into the second ("else") branch to make it conform. - // We continue respect the data structure semantic guarantees of the else_branch so - // that we can use all the code emitting abstractions. This is why at the bottom we - // assert that parent_branch.free_registers equals the saved_then_branch.free_registers - // rather than assigning it. - const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2]; - try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count()); - - const else_slice = else_branch.inst_table.entries.slice(); - const else_keys = else_slice.items(.key); - const else_values = else_slice.items(.value); - for (else_keys, 0..) |else_key, else_idx| { - const else_value = else_values[else_idx]; - const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) |then_entry| blk: { - // The instruction's MCValue is overridden in both branches. - log.debug("condBr put branch table (key = %{d}, value = {})", .{ else_key, then_entry.value }); - parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value); - if (else_value == .dead) { - assert(then_entry.value == .dead); - continue; - } - break :blk then_entry.value; - } else blk: { - if (else_value == .dead) - continue; - // The instruction is only overridden in the else branch. - var i: usize = self.branch_stack.items.len - 2; - while (true) { - i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead? - if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| { - assert(mcv != .dead); - break :blk mcv; - } - } - }; - log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv }); - // TODO make sure the destination stack offset / register does not already have something - // going on there. - try self.setRegOrMem(self.typeOfIndex(else_key), canon_mcv, else_value); - // TODO track the new register / stack allocation - } - try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count()); - const then_slice = saved_then_branch.inst_table.entries.slice(); - const then_keys = then_slice.items(.key); - const then_values = then_slice.items(.value); - for (then_keys, 0..) |then_key, then_idx| { - const then_value = then_values[then_idx]; - // We already deleted the items from this table that matched the else_branch. - // So these are all instructions that are only overridden in the then branch. - parent_branch.inst_table.putAssumeCapacity(then_key, then_value); - if (then_value == .dead) - continue; - const parent_mcv = blk: { - var i: usize = self.branch_stack.items.len - 2; - while (true) { - i -= 1; - if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| { - assert(mcv != .dead); - break :blk mcv; - } - } - }; - log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value }); - // TODO make sure the destination stack offset / register does not already have something - // going on there. - try self.setRegOrMem(self.typeOfIndex(then_key), parent_mcv, then_value); - // TODO track the new register / stack allocation - } - - { - var item = self.branch_stack.pop().?; - item.deinit(self.gpa); - } - - // We already took care of pl_op.operand earlier, so we're going - // to pass .none here - return self.finishAir(inst, .unreach, .{ .none, .none, .none }); -} - -fn airCtz(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); - // TODO emit debug info for function change - try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); -} - -fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void { - const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; - - _ = try self.addInst(.{ - .tag = .dbg_line, - .data = .{ - .dbg_line_column = .{ - .line = dbg_stmt.line, - .column = dbg_stmt.column, - }, - }, - }); - - return self.finishAirBookkeeping(); -} - -fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); - const operand = pl_op.operand; - // TODO emit debug info for this variable - _ = name; - return self.finishAir(inst, .dead, .{ operand, .none, .none }); -} - -fn airDiv(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airErrorName(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else { - _ = operand; - return self.fail("TODO implement airErrorName for {}", .{self.target.cpu.arch}); - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntFromFloat for {}", .{ - self.target.cpu.arch, - }); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - if (self.liveness.isUnused(inst)) - return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); - - const pt = self.pt; - const zcu = pt.zcu; - const operand_ty = self.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); - const info_a = operand_ty.intInfo(zcu); - const info_b = self.typeOfIndex(inst).intInfo(zcu); - if (info_a.signedness != info_b.signedness) - return self.fail("TODO gen intcast sign safety in semantic analysis", .{}); - - if (info_a.bits == info_b.bits) - return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none }); - - return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch}); -} - -fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatFromInt for {}", .{ - self.target.cpu.arch, - }); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIsErr(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(un_op); - const ty = self.typeOf(un_op); - break :result try self.isErr(ty, operand); - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(un_op); - const ty = self.typeOf(un_op); - break :result try self.isNonErr(ty, operand); - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airIsNull(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(un_op); - break :result try self.isNull(operand); - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(un_op); - break :result try self.isNonNull(operand); - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airLoad(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const elem_ty = self.typeOfIndex(inst); - const elem_size = elem_ty.abiSize(zcu); - const result: MCValue = result: { - if (!elem_ty.hasRuntimeBits(zcu)) - break :result MCValue.none; - - const ptr = try self.resolveInst(ty_op.operand); - const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(zcu); - if (self.liveness.isUnused(inst) and !is_volatile) - break :result MCValue.dead; - - const dst_mcv: MCValue = blk: { - if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) { - // The MCValue that holds the pointer can be re-used as the value. - break :blk switch (ptr) { - .register => |r| MCValue{ .register = r }, - else => ptr, - }; - } else { - break :blk try self.allocRegOrMem(inst, true); - } - }; - try self.load(dst_mcv, ptr, self.typeOf(ty_op.operand)); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airLoop(self: *Self, inst: Air.Inst.Index) !void { - // A loop is a setup to be able to jump back to the beginning. - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end .. loop.end + loop.data.body_len]); - const start: u32 = @intCast(self.mir_instructions.len); - - try self.genBody(body); - try self.jump(start); - - return self.finishAirBookkeeping(); -} - -fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { - if (safety) { - // TODO if the value is undef, write 0xaa bytes to dest - } else { - // TODO if the value is undef, don't lower this instruction - } - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload); - - const operand = pl_op.operand; - const value = extra.data.lhs; - const length = extra.data.rhs; - _ = operand; - _ = value; - _ = length; - - return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch}); -} - -fn airMinMax(self: *Self, inst: Air.Inst.Index) !void { - const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.minMax(tag, lhs, rhs, lhs_ty, rhs_ty); - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMod(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - assert(lhs_ty.eql(rhs_ty, self.pt.zcu)); - - if (self.liveness.isUnused(inst)) - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - - // TODO add safety check - - // We use manual assembly emission to generate faster code - // First, ensure lhs, rhs, rem, and added are in registers - - const lhs_is_register = lhs == .register; - const rhs_is_register = rhs == .register; - - const lhs_reg = if (lhs_is_register) - lhs.register - else - try self.register_manager.allocReg(null, gp); - - const lhs_lock = self.register_manager.lockReg(lhs_reg); - defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); - - const rhs_reg = if (rhs_is_register) - rhs.register - else - try self.register_manager.allocReg(null, gp); - const rhs_lock = self.register_manager.lockReg(rhs_reg); - defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); - - if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); - if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); - - const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp); - const regs_locks = self.register_manager.lockRegsAssumeUnused(2, regs); - defer for (regs_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const add_reg = regs[0]; - const mod_reg = regs[1]; - - // mod_reg = @rem(lhs_reg, rhs_reg) - _ = try self.addInst(.{ - .tag = .sdivx, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = mod_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .mulx, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = mod_reg, - .rs1 = mod_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .sub, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = mod_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .rs2 = mod_reg }, - }, - }, - }); - - // add_reg = mod_reg + rhs_reg - _ = try self.addInst(.{ - .tag = .add, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = add_reg, - .rs1 = mod_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - }); - - // if (add_reg == rhs_reg) add_reg = 0 - _ = try self.addInst(.{ - .tag = .cmp, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = add_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .movcc, - .data = .{ - .conditional_move_int = .{ - .is_imm = true, - .ccr = .xcc, - .cond = .{ .icond = .eq }, - .rd = add_reg, - .rs2_or_imm = .{ .imm = 0 }, - }, - }, - }); - - // if (lhs_reg < 0) mod_reg = add_reg - _ = try self.addInst(.{ - .tag = .movr, - .data = .{ - .conditional_move_reg = .{ - .is_imm = false, - .cond = .lt_zero, - .rd = mod_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .rs2 = add_reg }, - }, - }, - }); - - return self.finishAir(inst, .{ .register = mod_reg }, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { - //const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - const pt = self.pt; - const zcu = pt.zcu; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const lhs = try self.resolveInst(extra.lhs); - const rhs = try self.resolveInst(extra.rhs); - const lhs_ty = self.typeOf(extra.lhs); - const rhs_ty = self.typeOf(extra.rhs); - - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO implement mul_with_overflow for vectors", .{}), - .int => { - assert(lhs_ty.eql(rhs_ty, zcu)); - const int_info = lhs_ty.intInfo(zcu); - switch (int_info.bits) { - 1...32 => { - try self.spillConditionFlagsIfOccupied(); - - const dest = try self.binOp(.mul, lhs, rhs, lhs_ty, rhs_ty, null); - - const dest_reg = dest.register; - const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); - defer self.register_manager.unlockReg(dest_reg_lock); - - const truncated_reg = try self.register_manager.allocReg(null, gp); - const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); - defer self.register_manager.unlockReg(truncated_reg_lock); - - try self.truncRegister( - dest_reg, - truncated_reg, - int_info.signedness, - int_info.bits, - ); - - _ = try self.addInst(.{ - .tag = .cmp, - .data = .{ .arithmetic_2op = .{ - .is_imm = false, - .rs1 = dest_reg, - .rs2_or_imm = .{ .rs2 = truncated_reg }, - } }, - }); - - const cond = Instruction.ICondition.ne; - const ccr = Instruction.CCR.xcc; - - break :result MCValue{ .register_with_overflow = .{ - .reg = truncated_reg, - .flag = .{ .cond = cond, .ccr = ccr }, - } }; - }, - // XXX DO NOT call __multi3 directly as it'll result in us doing six multiplications, - // which is far more than strictly necessary - 33...64 => return self.fail("TODO copy compiler-rt's mulddi3 for a 64x64->128 multiply", .{}), - else => return self.fail("TODO overflow operations on other integer sizes", .{}), - } - }, - else => unreachable, - } - }; - return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airNot(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const pt = self.pt; - const zcu = pt.zcu; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - switch (operand) { - .dead => unreachable, - .unreach => unreachable, - .condition_flags => |op| { - break :result MCValue{ - .condition_flags = .{ - .cond = op.cond.negate(), - .ccr = op.ccr, - }, - }; - }, - else => { - switch (operand_ty.zigTypeTag(zcu)) { - .bool => { - const op_reg = switch (operand) { - .register => |r| r, - else => try self.copyToTmpRegister(operand_ty, operand), - }; - const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg); - defer self.register_manager.unlockReg(reg_lock); - - const dest_reg = blk: { - if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { - break :blk op_reg; - } - - const reg = try self.register_manager.allocReg(null, gp); - break :blk reg; - }; - - _ = try self.addInst(.{ - .tag = .xor, - .data = .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = op_reg, - .rs2_or_imm = .{ .imm = 1 }, - }, - }, - }); - - break :result MCValue{ .register = dest_reg }; - }, - .vector => return self.fail("TODO bitwise not for vectors", .{}), - .int => { - const int_info = operand_ty.intInfo(zcu); - if (int_info.bits <= 64) { - const op_reg = switch (operand) { - .register => |r| r, - else => try self.copyToTmpRegister(operand_ty, operand), - }; - const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg); - defer self.register_manager.unlockReg(reg_lock); - - const dest_reg = blk: { - if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { - break :blk op_reg; - } - - const reg = try self.register_manager.allocReg(null, gp); - break :blk reg; - }; - - _ = try self.addInst(.{ - .tag = .not, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = dest_reg, - .rs2_or_imm = .{ .rs2 = op_reg }, - }, - }, - }); - - try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits); - - break :result MCValue{ .register = dest_reg }; - } else { - return self.fail("TODO sparc64 not on integers > u64/i64", .{}); - } - }, - else => unreachable, - } - }, - } - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void { - const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; - // TODO Emit a PREFETCH/IPREFETCH as necessary, see A.7 and A.42 - return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none }); -} - -fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { - const is_volatile = false; // TODO - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const ptr_bits = self.target.ptrBitWidth(); - const ptr_bytes = @divExact(ptr_bits, 8); - const mcv = try self.resolveInst(ty_op.operand); - switch (mcv) { - .dead, .unreach, .none => unreachable, - .ptr_stack_offset => |off| { - break :result MCValue{ .ptr_stack_offset = off - ptr_bytes }; - }, - else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}), - } - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const mcv = try self.resolveInst(ty_op.operand); - switch (mcv) { - .dead, .unreach, .none => unreachable, - .ptr_stack_offset => |off| { - break :result MCValue{ .ptr_stack_offset = off }; - }, - else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}), - } - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airRem(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - - // TODO add safety check - - // result = lhs - @divTrunc(lhs, rhs) * rhs - const result: MCValue = if (self.liveness.isUnused(inst)) blk: { - break :blk .dead; - } else blk: { - const tmp0 = try self.binOp(.div_trunc, lhs, rhs, lhs_ty, rhs_ty, null); - const tmp1 = try self.binOp(.mul, tmp0, rhs, lhs_ty, rhs_ty, null); - break :blk try self.binOp(.sub, lhs, tmp1, lhs_ty, rhs_ty, null); - }; - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airRet(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - try self.ret(operand); - return self.finishAir(inst, .dead, .{ un_op, .none, .none }); -} - -fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ptr = try self.resolveInst(un_op); - _ = ptr; - return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch}); - //return self.finishAir(inst, .dead, .{ un_op, .none, .none }); -} - -fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { - const stack_offset = try self.allocMemPtr(inst); - return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none }); -} - -fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - _ = bin_op; - return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch}); -} - -fn airShlSat(self: *Self, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else if (self.typeOf(bin_op.lhs).isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.fail("TODO implement vector shl_sat with scalar rhs for {}", .{self.target.cpu.arch}) - else - return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - const pt = self.pt; - const zcu = pt.zcu; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const lhs = try self.resolveInst(extra.lhs); - const rhs = try self.resolveInst(extra.rhs); - const lhs_ty = self.typeOf(extra.lhs); - const rhs_ty = self.typeOf(extra.rhs); - - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => if (!rhs_ty.isVector(zcu)) - return self.fail("TODO implement vector shl_with_overflow with scalar rhs", .{}) - else - return self.fail("TODO implement mul_with_overflow for vectors", .{}), - .int => { - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - try self.spillConditionFlagsIfOccupied(); - - const lhs_lock: ?RegisterLock = if (lhs == .register) - self.register_manager.lockRegAssumeUnused(lhs.register) - else - null; - // TODO this currently crashes stage1 - // defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); - - // Increase shift amount (i.e, rhs) by shamt_bits - int_info.bits - // e.g if shifting a i48 then use sr*x (shamt_bits == 64) but increase rhs by 16 - // and if shifting a i24 then use sr* (shamt_bits == 32) but increase rhs by 8 - const new_rhs = switch (int_info.bits) { - 1...31 => if (rhs == .immediate) MCValue{ - .immediate = rhs.immediate + 32 - int_info.bits, - } else try self.binOp(.add, rhs, .{ .immediate = 32 - int_info.bits }, rhs_ty, rhs_ty, null), - 33...63 => if (rhs == .immediate) MCValue{ - .immediate = rhs.immediate + 64 - int_info.bits, - } else try self.binOp(.add, rhs, .{ .immediate = 64 - int_info.bits }, rhs_ty, rhs_ty, null), - 32, 64 => rhs, - else => unreachable, - }; - - const new_rhs_lock: ?RegisterLock = if (new_rhs == .register) - self.register_manager.lockRegAssumeUnused(new_rhs.register) - else - null; - // TODO this currently crashes stage1 - // defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); - - const dest = try self.binOp(.shl, lhs, new_rhs, lhs_ty, rhs_ty, null); - const dest_reg = dest.register; - const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); - defer self.register_manager.unlockReg(dest_reg_lock); - - const shr = try self.binOp(.shr, dest, new_rhs, lhs_ty, rhs_ty, null); - - _ = try self.addInst(.{ - .tag = .cmp, - .data = .{ .arithmetic_2op = .{ - .is_imm = false, - .rs1 = dest_reg, - .rs2_or_imm = .{ .rs2 = shr.register }, - } }, - }); - - const cond = Instruction.ICondition.ne; - const ccr = switch (int_info.bits) { - 1...32 => Instruction.CCR.icc, - 33...64 => Instruction.CCR.xcc, - else => unreachable, - }; - - // TODO Those should really be written as defers, however stage1 currently - // panics when those are turned into defer statements so those are - // written here at the end as ordinary statements. - // Because of that, on failure, the lock on those registers wouldn't be - // released. - if (lhs_lock) |reg| self.register_manager.unlockReg(reg); - if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); - - break :result MCValue{ .register_with_overflow = .{ - .reg = dest_reg, - .flag = .{ .cond = cond, .ccr = ccr }, - } }; - } else { - return self.fail("TODO overflow operations on other integer sizes", .{}); - } - }, - else => unreachable, - } - }; - return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airSlice(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const ptr = try self.resolveInst(bin_op.lhs); - const ptr_ty = self.typeOf(bin_op.lhs); - const len = try self.resolveInst(bin_op.rhs); - const len_ty = self.typeOf(bin_op.rhs); - const ptr_bytes = 8; - const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8"); - try self.genSetStack(ptr_ty, stack_offset, ptr); - try self.genSetStack(len_ty, stack_offset - ptr_bytes, len); - break :result MCValue{ .stack_offset = stack_offset }; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const is_volatile = false; // TODO - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - const result: MCValue = result: { - const slice_mcv = try self.resolveInst(bin_op.lhs); - const index_mcv = try self.resolveInst(bin_op.rhs); - - const slice_ty = self.typeOf(bin_op.lhs); - const elem_ty = slice_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); - - const index_lock: ?RegisterLock = if (index_mcv == .register) - self.register_manager.lockRegAssumeUnused(index_mcv.register) - else - null; - defer if (index_lock) |reg| self.register_manager.unlockReg(reg); - - const base_mcv: MCValue = switch (slice_mcv) { - .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) }, - else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}), - }; - const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register); - defer self.register_manager.unlockReg(base_lock); - - switch (elem_size) { - else => { - // TODO skip the ptr_add emission entirely and use native addressing modes - // i.e sllx/mulx then R+R or scale immediate then R+I - const dest = try self.allocRegOrMem(inst, true); - const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ptr_field_type, Type.usize, null); - try self.load(dest, addr, slice_ptr_field_type); - - break :result dest; - }, - } - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const ptr_bits = self.target.ptrBitWidth(); - const ptr_bytes = @divExact(ptr_bits, 8); - const mcv = try self.resolveInst(ty_op.operand); - switch (mcv) { - .dead, .unreach, .none => unreachable, - .register => unreachable, // a slice doesn't fit in one register - .stack_offset => |off| { - break :result MCValue{ .stack_offset = off - ptr_bytes }; - }, - .memory => |addr| { - break :result MCValue{ .memory = addr + ptr_bytes }; - }, - else => return self.fail("TODO implement slice_len for {}", .{mcv}), - } - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const mcv = try self.resolveInst(ty_op.operand); - switch (mcv) { - .dead, .unreach, .none => unreachable, - .register => unreachable, // a slice doesn't fit in one register - .stack_offset => |off| { - break :result MCValue{ .stack_offset = off }; - }, - .memory => |addr| { - break :result MCValue{ .memory = addr }; - }, - else => return self.fail("TODO implement slice_len for {}", .{mcv}), - } - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSplat(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void { - if (safety) { - // TODO if the value is undef, write 0xaa bytes to dest - } else { - // TODO if the value is undef, don't lower this instruction - } - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ptr = try self.resolveInst(bin_op.lhs); - const value = try self.resolveInst(bin_op.rhs); - const ptr_ty = self.typeOf(bin_op.lhs); - const value_ty = self.typeOf(bin_op.rhs); - - try self.store(ptr, value, ptr_ty, value_ty); - - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; - const result = try self.structFieldPtr(inst, extra.struct_operand, extra.field_index); - return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); -} - -fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = try self.structFieldPtr(inst, ty_op.operand, index); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; - const operand = extra.struct_operand; - const index = extra.field_index; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const zcu = self.pt.zcu; - const mcv = try self.resolveInst(operand); - const struct_ty = self.typeOf(operand); - const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu)); - - switch (mcv) { - .dead, .unreach => unreachable, - .stack_offset => |off| { - break :result MCValue{ .stack_offset = off - struct_field_offset }; - }, - .memory => |addr| { - break :result MCValue{ .memory = addr + struct_field_offset }; - }, - .register_with_overflow => |rwo| { - switch (index) { - 0 => { - // get wrapped value: return register - break :result MCValue{ .register = rwo.reg }; - }, - 1 => { - // TODO return special MCValue condition flags - // get overflow bit: set register to C flag - // resp. V flag - const dest_reg = try self.register_manager.allocReg(null, gp); - - // TODO handle floating point CCRs - assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc); - - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = dest_reg, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .movcc, - .data = .{ - .conditional_move_int = .{ - .ccr = rwo.flag.ccr, - .cond = .{ .icond = rwo.flag.cond }, - .is_imm = true, - .rd = dest_reg, - .rs2_or_imm = .{ .imm = 1 }, - }, - }, - }); - - break :result MCValue{ .register = dest_reg }; - }, - else => unreachable, - } - }, - else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}), - } - }; - - return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); -} - -fn airSubSat(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { - _ = inst; - return self.fail("TODO implement switch for {}", .{self.target.cpu.arch}); -} - -fn airTagName(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else { - _ = operand; - return self.fail("TODO implement airTagName for {}", .{self.target.cpu.arch}); - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const dest_ty = self.typeOfIndex(inst); - - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: { - break :blk try self.trunc(inst, operand, operand_ty, dest_ty); - }; - - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airTry(self: *Self, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); - const result: MCValue = result: { - const error_union_ty = self.typeOf(pl_op.operand); - const error_union = try self.resolveInst(pl_op.operand); - const is_err_result = try self.isErr(error_union_ty, error_union); - const reloc = try self.condBr(is_err_result); - - try self.genBody(body); - - try self.performReloc(reloc); - break :result try self.errUnionPayload(error_union, error_union_ty); - }; - return self.finishAir(inst, result, .{ pl_op.operand, .none, .none }); -} - -fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; - _ = extra; - return self.fail("TODO implement airUnionInit for {}", .{self.target.cpu.arch}); -} - -fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const error_union_ty = self.typeOf(ty_op.operand); - const payload_ty = error_union_ty.errorUnionPayload(zcu); - const mcv = try self.resolveInst(ty_op.operand); - if (!payload_ty.hasRuntimeBits(zcu)) break :result mcv; - - return self.fail("TODO implement unwrap error union error for non-empty payloads", .{}); - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const error_union_ty = self.typeOf(ty_op.operand); - const payload_ty = error_union_ty.errorUnionPayload(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) break :result MCValue.none; - - return self.fail("TODO implement unwrap error union payload for non-empty payloads", .{}); - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -/// E to E!T -fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const error_union_ty = ty_op.ty.toType(); - const payload_ty = error_union_ty.errorUnionPayload(zcu); - const mcv = try self.resolveInst(ty_op.operand); - if (!payload_ty.hasRuntimeBits(zcu)) break :result mcv; - - return self.fail("TODO implement wrap errunion error for non-empty payloads", .{}); - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -/// T to E!T -fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const optional_ty = self.typeOfIndex(inst); - - // Optional with a zero-bit payload type is just a boolean true - if (optional_ty.abiSize(pt.zcu) == 1) - break :result MCValue{ .immediate = 1 }; - - return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch}); - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -// Common helper functions - -fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { - const gpa = self.gpa; - try self.mir_instructions.ensureUnusedCapacity(gpa, 1); - const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len); - self.mir_instructions.appendAssumeCapacity(inst); - return result_index; -} - -fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: Alignment) !u32 { - self.stack_align = self.stack_align.max(abi_align); - // TODO find a free slot instead of always appending - const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset) + abi_size); - self.next_stack_offset = offset; - if (self.next_stack_offset > self.max_end_stack) - self.max_end_stack = self.next_stack_offset; - try self.stack.putNoClobber(self.gpa, offset, .{ - .inst = inst, - .size = abi_size, - }); - return offset; -} - -/// Use a pointer instruction as the basis for allocating stack memory. -fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 { - const pt = self.pt; - const zcu = pt.zcu; - const elem_ty = self.typeOfIndex(inst).childType(zcu); - - if (!elem_ty.hasRuntimeBits(zcu)) { - // As this stack item will never be dereferenced at runtime, - // return the stack offset 0. Stack offset 0 will be where all - // zero-sized stack allocations live as non-zero-sized - // allocations will always have an offset > 0. - return @as(u32, 0); - } - - const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse { - return self.fail("type '{f}' too big to fit into stack frame", .{elem_ty.fmt(pt)}); - }; - // TODO swap this for inst.ty.ptrAlign - const abi_align = elem_ty.abiAlignment(zcu); - return self.allocMem(inst, abi_size, abi_align); -} - -fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const elem_ty = self.typeOfIndex(inst); - const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse { - return self.fail("type '{f}' too big to fit into stack frame", .{elem_ty.fmt(pt)}); - }; - const abi_align = elem_ty.abiAlignment(zcu); - self.stack_align = self.stack_align.max(abi_align); - - if (reg_ok) { - // Make sure the type can fit in a register before we try to allocate one. - if (abi_size <= 8) { - if (self.register_manager.tryAllocReg(inst, gp)) |reg| { - return MCValue{ .register = reg }; - } - } - } - const stack_offset = try self.allocMem(inst, abi_size, abi_align); - return MCValue{ .stack_offset = stack_offset }; -} - -const BinOpMetadata = struct { - inst: Air.Inst.Index, - lhs: Air.Inst.Ref, - rhs: Air.Inst.Ref, -}; - -/// For all your binary operation needs, this function will generate -/// the corresponding Mir instruction(s). Returns the location of the -/// result. -/// -/// If the binary operation itself happens to be an Air instruction, -/// pass the corresponding index in the inst parameter. That helps -/// this function do stuff like reusing operands. -/// -/// This function does not do any lowering to Mir itself, but instead -/// looks at the lhs and rhs and determines which kind of lowering -/// would be best suitable and then delegates the lowering to other -/// functions. -fn binOp( - self: *Self, - tag: Air.Inst.Tag, - lhs: MCValue, - rhs: MCValue, - lhs_ty: Type, - rhs_ty: Type, - metadata: ?BinOpMetadata, -) InnerError!MCValue { - const pt = self.pt; - const zcu = pt.zcu; - switch (tag) { - .add, - .sub, - .mul, - .bit_and, - .bit_or, - .xor, - .cmp_eq, - => { - switch (lhs_ty.zigTypeTag(zcu)) { - .float => return self.fail("TODO binary operations on floats", .{}), - .vector => return self.fail("TODO binary operations on vectors", .{}), - .int => { - assert(lhs_ty.eql(rhs_ty, zcu)); - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - // Only say yes if the operation is - // commutative, i.e. we can swap both of the - // operands - const lhs_immediate_ok = switch (tag) { - .add => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), - .mul => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), - .bit_and => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), - .bit_or => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), - .xor => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), - .sub, .cmp_eq => false, - else => unreachable, - }; - const rhs_immediate_ok = switch (tag) { - .add, - .sub, - .mul, - .bit_and, - .bit_or, - .xor, - .cmp_eq, - => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), - else => unreachable, - }; - - const mir_tag: Mir.Inst.Tag = switch (tag) { - .add => .add, - .sub => .sub, - .mul => .mulx, - .bit_and => .@"and", - .bit_or => .@"or", - .xor => .xor, - .cmp_eq => .cmp, - else => unreachable, - }; - - if (rhs_immediate_ok) { - return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata); - } else if (lhs_immediate_ok) { - // swap lhs and rhs - return try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, metadata); - } else { - // TODO convert large immediates to register before adding - return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - } - } else { - return self.fail("TODO binary operations on int with bits > 64", .{}); - } - }, - else => unreachable, - } - }, - - .add_wrap, - .sub_wrap, - .mul_wrap, - => { - const base_tag: Air.Inst.Tag = switch (tag) { - .add_wrap => .add, - .sub_wrap => .sub, - .mul_wrap => .mul, - else => unreachable, - }; - - // Generate the base operation - const result = try self.binOp(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - - // Truncate if necessary - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO binary operations on vectors", .{}), - .int => { - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - const result_reg = result.register; - try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits); - return result; - } else { - return self.fail("TODO binary operations on integers > u64/i64", .{}); - } - }, - else => unreachable, - } - }, - - .div_trunc => { - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO binary operations on vectors", .{}), - .int => { - assert(lhs_ty.eql(rhs_ty, zcu)); - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - const rhs_immediate_ok = switch (tag) { - .div_trunc => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), - else => unreachable, - }; - - const mir_tag: Mir.Inst.Tag = switch (tag) { - .div_trunc => switch (int_info.signedness) { - .signed => Mir.Inst.Tag.sdivx, - .unsigned => Mir.Inst.Tag.udivx, - }, - else => unreachable, - }; - - if (rhs_immediate_ok) { - return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, true, metadata); - } else { - return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - } - } else { - return self.fail("TODO binary operations on int with bits > 64", .{}); - } - }, - else => unreachable, - } - }, - - .ptr_add => { - switch (lhs_ty.zigTypeTag(zcu)) { - .pointer => { - const ptr_ty = lhs_ty; - const elem_ty = switch (ptr_ty.ptrSize(zcu)) { - .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type - else => ptr_ty.childType(zcu), - }; - const elem_size = elem_ty.abiSize(zcu); - - if (elem_size == 1) { - const base_tag: Mir.Inst.Tag = switch (tag) { - .ptr_add => .add, - else => unreachable, - }; - - return try self.binOpRegister(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - } else { - // convert the offset into a byte offset by - // multiplying it with elem_size - - const offset = try self.binOp(.mul, rhs, .{ .immediate = elem_size }, Type.usize, Type.usize, null); - const addr = try self.binOp(tag, lhs, offset, Type.manyptr_u8, Type.usize, null); - return addr; - } - }, - else => unreachable, - } - }, - - .bool_and, - .bool_or, - => { - switch (lhs_ty.zigTypeTag(zcu)) { - .bool => { - assert(lhs != .immediate); // should have been handled by Sema - assert(rhs != .immediate); // should have been handled by Sema - - const mir_tag: Mir.Inst.Tag = switch (tag) { - .bool_and => .@"and", - .bool_or => .@"or", - else => unreachable, - }; - - return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - }, - else => unreachable, - } - }, - - .shl, - .shr, - => { - const base_tag: Air.Inst.Tag = switch (tag) { - .shl => .shl_exact, - .shr => .shr_exact, - else => unreachable, - }; - - // Generate the base operation - const result = try self.binOp(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - - // Truncate if necessary - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => if (rhs_ty.isVector(zcu)) - return self.fail("TODO vector shift with scalar rhs", .{}) - else - return self.fail("TODO binary operations on vectors", .{}), - .int => { - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - // 32 and 64 bit operands doesn't need truncating - if (int_info.bits == 32 or int_info.bits == 64) return result; - - const result_reg = result.register; - try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits); - return result; - } else { - return self.fail("TODO binary operations on integers > u64/i64", .{}); - } - }, - else => unreachable, - } - }, - - .shl_exact, - .shr_exact, - => { - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => if (rhs_ty.isVector(zcu)) - return self.fail("TODO vector shift with scalar rhs", .{}) - else - return self.fail("TODO binary operations on vectors", .{}), - .int => { - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - const rhs_immediate_ok = rhs == .immediate; - - const mir_tag: Mir.Inst.Tag = switch (tag) { - .shl_exact => if (int_info.bits <= 32) Mir.Inst.Tag.sll else Mir.Inst.Tag.sllx, - .shr_exact => switch (int_info.signedness) { - .signed => if (int_info.bits <= 32) Mir.Inst.Tag.sra else Mir.Inst.Tag.srax, - .unsigned => if (int_info.bits <= 32) Mir.Inst.Tag.srl else Mir.Inst.Tag.srlx, - }, - else => unreachable, - }; - - if (rhs_immediate_ok) { - return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata); - } else { - return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); - } - } else { - return self.fail("TODO binary operations on int with bits > 64", .{}); - } - }, - else => unreachable, - } - }, - - else => return self.fail("TODO implement {} binOp for SPARCv9", .{tag}), - } -} - -/// Don't call this function directly. Use binOp instead. -/// -/// Calling this function signals an intention to generate a Mir -/// instruction of the form -/// -/// op dest, lhs, #rhs_imm -/// -/// Set lhs_and_rhs_swapped to true iff inst.bin_op.lhs corresponds to -/// rhs and vice versa. This parameter is only used when metadata != null. -/// -/// Asserts that generating an instruction of that form is possible. -fn binOpImmediate( - self: *Self, - mir_tag: Mir.Inst.Tag, - lhs: MCValue, - rhs: MCValue, - lhs_ty: Type, - lhs_and_rhs_swapped: bool, - metadata: ?BinOpMetadata, -) !MCValue { - const lhs_is_register = lhs == .register; - - const lhs_lock: ?RegisterLock = if (lhs_is_register) - self.register_manager.lockReg(lhs.register) - else - null; - defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); - - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - - const lhs_reg = if (lhs_is_register) lhs.register else blk: { - const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: { - break :inst (if (lhs_and_rhs_swapped) md.rhs else md.lhs).toIndex().?; - } else null; - - const reg = try self.register_manager.allocReg(track_inst, gp); - - if (track_inst) |inst| { - const mcv: MCValue = .{ .register = reg }; - log.debug("binOpRegister move lhs %{d} to register: {} -> {}", .{ inst, lhs, mcv }); - branch.inst_table.putAssumeCapacity(inst, mcv); - - // If we're moving a condition flag MCV to register, - // mark it as free. - if (lhs == .condition_flags) { - assert(self.condition_flags_inst.? == inst); - self.condition_flags_inst = null; - } - } - - break :blk reg; - }; - const new_lhs_lock = self.register_manager.lockReg(lhs_reg); - defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); - - const dest_reg = switch (mir_tag) { - .cmp => undefined, // cmp has no destination register - else => if (metadata) |md| blk: { - if (lhs_is_register and self.reuseOperand( - md.inst, - if (lhs_and_rhs_swapped) md.rhs else md.lhs, - if (lhs_and_rhs_swapped) 1 else 0, - lhs, - )) { - break :blk lhs_reg; - } else { - break :blk try self.register_manager.allocReg(md.inst, gp); - } - } else blk: { - break :blk try self.register_manager.allocReg(null, gp); - }, - }; - - if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); - - const mir_data: Mir.Inst.Data = switch (mir_tag) { - .add, - .addcc, - .@"and", - .@"or", - .xor, - .xnor, - .mulx, - .sdivx, - .udivx, - .sub, - .subcc, - => .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .imm = @as(u12, @intCast(rhs.immediate)) }, - }, - }, - .sll, - .srl, - .sra, - => .{ - .shift = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .imm = @as(u5, @intCast(rhs.immediate)) }, - }, - }, - .sllx, - .srlx, - .srax, - => .{ - .shift = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .imm = @as(u6, @intCast(rhs.immediate)) }, - }, - }, - .cmp => .{ - .arithmetic_2op = .{ - .is_imm = true, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .imm = @as(u12, @intCast(rhs.immediate)) }, - }, - }, - else => unreachable, - }; - - _ = try self.addInst(.{ - .tag = mir_tag, - .data = mir_data, - }); - - return MCValue{ .register = dest_reg }; -} - -/// Don't call this function directly. Use binOp instead. -/// -/// Calling this function signals an intention to generate a Mir -/// instruction of the form -/// -/// op dest, lhs, rhs -/// -/// Asserts that generating an instruction of that form is possible. -fn binOpRegister( - self: *Self, - mir_tag: Mir.Inst.Tag, - lhs: MCValue, - rhs: MCValue, - lhs_ty: Type, - rhs_ty: Type, - metadata: ?BinOpMetadata, -) !MCValue { - const lhs_is_register = lhs == .register; - const rhs_is_register = rhs == .register; - - const lhs_lock: ?RegisterLock = if (lhs_is_register) - self.register_manager.lockReg(lhs.register) - else - null; - defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); - - const rhs_lock: ?RegisterLock = if (rhs_is_register) - self.register_manager.lockReg(rhs.register) - else - null; - defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); - - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - - const lhs_reg = if (lhs_is_register) lhs.register else blk: { - const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: { - break :inst md.lhs.toIndex().?; - } else null; - - const reg = try self.register_manager.allocReg(track_inst, gp); - if (track_inst) |inst| { - const mcv: MCValue = .{ .register = reg }; - log.debug("binOpRegister move lhs %{d} to register: {} -> {}", .{ inst, lhs, mcv }); - branch.inst_table.putAssumeCapacity(inst, mcv); - - // If we're moving a condition flag MCV to register, - // mark it as free. - if (lhs == .condition_flags) { - assert(self.condition_flags_inst.? == inst); - self.condition_flags_inst = null; - } - } - - break :blk reg; - }; - const new_lhs_lock = self.register_manager.lockReg(lhs_reg); - defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); - - const rhs_reg = if (rhs_is_register) rhs.register else blk: { - const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: { - break :inst md.rhs.toIndex().?; - } else null; - - const reg = try self.register_manager.allocReg(track_inst, gp); - if (track_inst) |inst| { - const mcv: MCValue = .{ .register = reg }; - log.debug("binOpRegister move rhs %{d} to register: {} -> {}", .{ inst, rhs, mcv }); - branch.inst_table.putAssumeCapacity(inst, mcv); - - // If we're moving a condition flag MCV to register, - // mark it as free. - if (rhs == .condition_flags) { - assert(self.condition_flags_inst.? == inst); - self.condition_flags_inst = null; - } - } - - break :blk reg; - }; - const new_rhs_lock = self.register_manager.lockReg(rhs_reg); - defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); - - const dest_reg = switch (mir_tag) { - .cmp => undefined, // cmp has no destination register - else => if (metadata) |md| blk: { - if (lhs_is_register and self.reuseOperand(md.inst, md.lhs, 0, lhs)) { - break :blk lhs_reg; - } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { - break :blk rhs_reg; - } else { - break :blk try self.register_manager.allocReg(md.inst, gp); - } - } else blk: { - break :blk try self.register_manager.allocReg(null, gp); - }, - }; - - if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); - if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); - - const mir_data: Mir.Inst.Data = switch (mir_tag) { - .add, - .addcc, - .@"and", - .@"or", - .xor, - .xnor, - .mulx, - .sdivx, - .udivx, - .sub, - .subcc, - => .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = dest_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - .sll, - .srl, - .sra, - .sllx, - .srlx, - .srax, - => .{ - .shift = .{ - .is_imm = false, - .rd = dest_reg, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - .cmp => .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = lhs_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - else => unreachable, - }; - - _ = try self.addInst(.{ - .tag = mir_tag, - .data = mir_data, - }); - - return MCValue{ .register = dest_reg }; -} - -fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { - const block_data = self.blocks.getPtr(block).?; - - const zcu = self.pt.zcu; - if (self.typeOf(operand).hasRuntimeBits(zcu)) { - const operand_mcv = try self.resolveInst(operand); - const block_mcv = block_data.mcv; - if (block_mcv == .none) { - block_data.mcv = switch (operand_mcv) { - .none, .dead, .unreach => unreachable, - .register, .stack_offset, .memory => operand_mcv, - .immediate => blk: { - const new_mcv = try self.allocRegOrMem(block, true); - try self.setRegOrMem(self.typeOfIndex(block), new_mcv, operand_mcv); - break :blk new_mcv; - }, - else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), - }; - } else { - try self.setRegOrMem(self.typeOfIndex(block), block_mcv, operand_mcv); - } - } - return self.brVoid(block); -} - -fn brVoid(self: *Self, block: Air.Inst.Index) !void { - const block_data = self.blocks.getPtr(block).?; - - // Emit a jump with a relocation. It will be patched up after the block ends. - try block_data.relocs.ensureUnusedCapacity(self.gpa, 1); - - const br_index = try self.addInst(.{ - .tag = .bpcc, - .data = .{ - .branch_predict_int = .{ - .ccr = .xcc, - .cond = .al, - .inst = undefined, // Will be filled by performReloc - }, - }, - }); - - // TODO Find a way to fill this delay slot - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); - - block_data.relocs.appendAssumeCapacity(br_index); -} - -fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index { - // Here we either emit a BPcc for branching on CCR content, - // or emit a BPr to branch on register content. - const reloc: Mir.Inst.Index = switch (condition) { - .condition_flags => |flags| try self.addInst(.{ - .tag = .bpcc, - .data = .{ - .branch_predict_int = .{ - .ccr = flags.ccr, - // Here we map to the opposite condition because the jump is to the false branch. - .cond = flags.cond.icond.negate(), - .inst = undefined, // Will be filled by performReloc - }, - }, - }), - .condition_register => |reg| try self.addInst(.{ - .tag = .bpr, - .data = .{ - .branch_predict_reg = .{ - .rs1 = reg.reg, - // Here we map to the opposite condition because the jump is to the false branch. - .cond = reg.cond.negate(), - .inst = undefined, // Will be filled by performReloc - }, - }, - }), - else => blk: { - const reg = switch (condition) { - .register => |r| r, - else => try self.copyToTmpRegister(Type.bool, condition), - }; - - break :blk try self.addInst(.{ - .tag = .bpr, - .data = .{ - .branch_predict_reg = .{ - .cond = .eq_zero, - .rs1 = reg, - .inst = undefined, // populated later through performReloc - }, - }, - }); - }, - }; - - // Regardless of the branch type that's emitted, we need to reserve - // a space for the delay slot. - // TODO Find a way to fill this delay slot - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); - - return reloc; -} - -/// Copies a value to a register without tracking the register. The register is not considered -/// allocated. A second call to `copyToTmpRegister` may return the same register. -/// This can have a side effect of spilling instructions to the stack to free up a register. -fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null, gp); - try self.genSetReg(ty, reg, mcv); - return reg; -} - -fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void { - const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table; - try table.ensureUnusedCapacity(self.gpa, additional_count); -} - -/// Given an error union, returns the payload -fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const err_ty = error_union_ty.errorUnionSet(zcu); - const payload_ty = error_union_ty.errorUnionPayload(zcu); - if (err_ty.errorSetIsEmpty(zcu)) { - return error_union_mcv; - } - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return MCValue.none; - } - - const payload_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); - switch (error_union_mcv) { - .register => return self.fail("TODO errUnionPayload for registers", .{}), - .stack_offset => |off| { - return MCValue{ .stack_offset = off - payload_offset }; - }, - .memory => |addr| { - return MCValue{ .memory = addr + payload_offset }; - }, - else => unreachable, // invalid MCValue for an error union - } -} - -fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = self.pt.zcu; - const func = zcu.funcInfo(self.func_index); - const msg = try ErrorMsg.create(zcu.gpa, self.src_loc, format, args); - return zcu.codegenFailMsg(func.owner_nav, msg); -} - -fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = self.pt.zcu; - const func = zcu.funcInfo(self.func_index); - return zcu.codegenFailMsg(func.owner_nav, msg); -} - -/// Called when there are no operands, and the instruction is always unreferenced. -fn finishAirBookkeeping(self: *Self) void { - if (std.debug.runtime_safety) { - self.air_bookkeeping += 1; - } -} - -fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void { - const tomb_bits = self.liveness.getTombBits(inst); - for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; - if (self.reused_operands.isSet(op_index)) continue; - self.processDeath(op.toIndexAllowNone() orelse continue); - } - if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) { - log.debug("%{d} => {}", .{ inst, result }); - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - branch.inst_table.putAssumeCapacityNoClobber(inst, result); - - switch (result) { - .register => |reg| { - // In some cases (such as bitcast), an operand - // may be the same MCValue as the result. If - // that operand died and was a register, it - // was freed by processDeath. We have to - // "re-allocate" the register. - if (self.register_manager.isRegFree(reg)) { - self.register_manager.getRegAssumeFree(reg, inst); - } - }, - else => {}, - } - } - self.finishAirBookkeeping(); -} - -fn genArgDbgInfo(self: Self, name: []const u8, ty: Type, mcv: MCValue) !void { - // TODO: Add a pseudo-instruction or something to defer this work until Emit. - // We aren't allowed to interact with linker state here. - if (true) return; - switch (self.debug_output) { - .dwarf => |dw| switch (mcv) { - .register => |reg| try dw.genLocalDebugInfo( - .local_arg, - name, - ty, - .{ .reg = reg.dwarfNum() }, - ), - else => {}, - }, - else => {}, - } -} - -// TODO replace this to call to extern memcpy -fn genInlineMemcpy( - self: *Self, - src: Register, - dst: Register, - len: Register, - tmp: Register, -) !void { - // Here we assume that len > 0. - // Also we do the copy from end -> start address to save a register. - - // sub len, 1, len - _ = try self.addInst(.{ - .tag = .sub, - .data = .{ .arithmetic_3op = .{ - .is_imm = true, - .rs1 = len, - .rs2_or_imm = .{ .imm = 1 }, - .rd = len, - } }, - }); - - // loop: - // ldub [src + len], tmp - _ = try self.addInst(.{ - .tag = .ldub, - .data = .{ .arithmetic_3op = .{ - .is_imm = false, - .rs1 = src, - .rs2_or_imm = .{ .rs2 = len }, - .rd = tmp, - } }, - }); - - // stb tmp, [dst + len] - _ = try self.addInst(.{ - .tag = .stb, - .data = .{ .arithmetic_3op = .{ - .is_imm = false, - .rs1 = dst, - .rs2_or_imm = .{ .rs2 = len }, - .rd = tmp, - } }, - }); - - // brnz len, loop - _ = try self.addInst(.{ - .tag = .bpr, - .data = .{ .branch_predict_reg = .{ - .cond = .ne_zero, - .rs1 = len, - .inst = @as(u32, @intCast(self.mir_instructions.len - 2)), - } }, - }); - - // Delay slot: - // sub len, 1, len - _ = try self.addInst(.{ - .tag = .sub, - .data = .{ .arithmetic_3op = .{ - .is_imm = true, - .rs1 = len, - .rs2_or_imm = .{ .imm = 1 }, - .rd = len, - } }, - }); - - // end: -} - -fn genLoad(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void { - assert(off_type == Register or off_type == i13); - - const is_imm = (off_type == i13); - - switch (abi_size) { - 1, 2, 4, 8 => { - const tag: Mir.Inst.Tag = switch (abi_size) { - 1 => .ldub, - 2 => .lduh, - 4 => .lduw, - 8 => .ldx, - else => unreachable, // unexpected abi size - }; - - _ = try self.addInst(.{ - .tag = tag, - .data = .{ - .arithmetic_3op = .{ - .is_imm = is_imm, - .rd = value_reg, - .rs1 = addr_reg, - .rs2_or_imm = if (is_imm) .{ .imm = off } else .{ .rs2 = off }, - }, - }, - }); - }, - 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), - else => unreachable, - } -} - -fn genLoadASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void { - switch (abi_size) { - 1, 2, 4, 8 => { - const tag: Mir.Inst.Tag = switch (abi_size) { - 1 => .lduba, - 2 => .lduha, - 4 => .lduwa, - 8 => .ldxa, - else => unreachable, // unexpected abi size - }; - - _ = try self.addInst(.{ - .tag = tag, - .data = .{ - .mem_asi = .{ - .rd = value_reg, - .rs1 = addr_reg, - .rs2 = off_reg, - .asi = asi, - }, - }, - }); - }, - 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), - else => unreachable, - } -} - -fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - switch (mcv) { - .dead => unreachable, - .unreach, .none => return, // Nothing to do. - .condition_flags => |op| { - const condition = op.cond; - const ccr = op.ccr; - - // TODO handle floating point CCRs - assert(ccr == .xcc or ccr == .icc); - - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = reg, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .movcc, - .data = .{ - .conditional_move_int = .{ - .ccr = ccr, - .cond = condition, - .is_imm = true, - .rd = reg, - .rs2_or_imm = .{ .imm = 1 }, - }, - }, - }); - }, - .condition_register => |op| { - const condition = op.cond; - const register = op.reg; - - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = reg, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .movr, - .data = .{ - .conditional_move_reg = .{ - .cond = condition, - .is_imm = true, - .rd = reg, - .rs1 = register, - .rs2_or_imm = .{ .imm = 1 }, - }, - }, - }); - }, - .undef => { - if (!self.wantSafety()) - return; // The already existing value will do just fine. - // Write the debug undefined value. - return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }); - }, - .ptr_stack_offset => |off| { - const real_offset = realStackOffset(off); - const simm13 = math.cast(i13, real_offset) orelse - return self.fail("TODO larger stack offsets: {}", .{real_offset}); - - _ = try self.addInst(.{ - .tag = .add, - .data = .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = reg, - .rs1 = .sp, - .rs2_or_imm = .{ .imm = simm13 }, - }, - }, - }); - }, - .immediate => |x| { - if (x <= math.maxInt(u12)) { - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = true, - .rs1 = reg, - .rs2_or_imm = .{ .imm = @as(u12, @truncate(x)) }, - }, - }, - }); - } else if (x <= math.maxInt(u32)) { - _ = try self.addInst(.{ - .tag = .sethi, - .data = .{ - .sethi = .{ - .rd = reg, - .imm = @as(u22, @truncate(x >> 10)), - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .@"or", - .data = .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = reg, - .rs1 = reg, - .rs2_or_imm = .{ .imm = @as(u10, @truncate(x)) }, - }, - }, - }); - } else if (x <= math.maxInt(u44)) { - try self.genSetReg(ty, reg, .{ .immediate = @as(u32, @truncate(x >> 12)) }); - - _ = try self.addInst(.{ - .tag = .sllx, - .data = .{ - .shift = .{ - .is_imm = true, - .rd = reg, - .rs1 = reg, - .rs2_or_imm = .{ .imm = 12 }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .@"or", - .data = .{ - .arithmetic_3op = .{ - .is_imm = true, - .rd = reg, - .rs1 = reg, - .rs2_or_imm = .{ .imm = @as(u12, @truncate(x)) }, - }, - }, - }); - } else { - // Need to allocate a temporary register to load 64-bit immediates. - const tmp_reg = try self.register_manager.allocReg(null, gp); - - try self.genSetReg(ty, tmp_reg, .{ .immediate = @as(u32, @truncate(x)) }); - try self.genSetReg(ty, reg, .{ .immediate = @as(u32, @truncate(x >> 32)) }); - - _ = try self.addInst(.{ - .tag = .sllx, - .data = .{ - .shift = .{ - .is_imm = true, - .rd = reg, - .rs1 = reg, - .rs2_or_imm = .{ .imm = 32 }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .@"or", - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = reg, - .rs1 = reg, - .rs2_or_imm = .{ .rs2 = tmp_reg }, - }, - }, - }); - } - }, - .register => |src_reg| { - // If the registers are the same, nothing to do. - if (src_reg.id() == reg.id()) - return; - - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = reg, - .rs2_or_imm = .{ .rs2 = src_reg }, - }, - }, - }); - }, - .register_with_overflow => unreachable, - .memory => |addr| { - // The value is in memory at a hard-coded address. - // If the type is a pointer, it means the pointer address is at this memory location. - try self.genSetReg(ty, reg, .{ .immediate = addr }); - try self.genLoad(reg, reg, i13, 0, ty.abiSize(zcu)); - }, - .stack_offset => |off| { - const real_offset = realStackOffset(off); - const simm13 = math.cast(i13, real_offset) orelse - return self.fail("TODO larger stack offsets: {}", .{real_offset}); - try self.genLoad(reg, .sp, i13, simm13, ty.abiSize(zcu)); - }, - } -} - -fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size = ty.abiSize(zcu); - switch (mcv) { - .dead => unreachable, - .unreach, .none => return, // Nothing to do. - .undef => { - if (!self.wantSafety()) - return; // The already existing value will do just fine. - // TODO Upgrade this to a memset call when we have that available. - switch (ty.abiSize(zcu)) { - 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }), - 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }), - 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }), - 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), - else => return self.fail("TODO implement memset", .{}), - } - }, - .condition_flags, - .condition_register, - .immediate, - .ptr_stack_offset, - => { - const reg = try self.copyToTmpRegister(ty, mcv); - return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }); - }, - .register => |reg| { - const real_offset = realStackOffset(stack_offset); - const simm13 = math.cast(i13, real_offset) orelse - return self.fail("TODO larger stack offsets: {}", .{real_offset}); - return self.genStore(reg, .sp, i13, simm13, abi_size); - }, - .register_with_overflow => |rwo| { - const reg_lock = self.register_manager.lockReg(rwo.reg); - defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg); - - const wrapped_ty = ty.fieldType(0, zcu); - try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg }); - - const overflow_bit_ty = ty.fieldType(1, zcu); - const overflow_bit_offset: u32 = @intCast(ty.structFieldOffset(1, zcu)); - const cond_reg = try self.register_manager.allocReg(null, gp); - - // TODO handle floating point CCRs - assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc); - - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = cond_reg, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .movcc, - .data = .{ - .conditional_move_int = .{ - .ccr = rwo.flag.ccr, - .cond = .{ .icond = rwo.flag.cond }, - .is_imm = true, - .rd = cond_reg, - .rs2_or_imm = .{ .imm = 1 }, - }, - }, - }); - try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{ - .register = cond_reg, - }); - }, - .memory, .stack_offset => { - switch (mcv) { - .stack_offset => |off| { - if (stack_offset == off) - return; // Copy stack variable to itself; nothing to do. - }, - else => {}, - } - - if (abi_size <= 8) { - const reg = try self.copyToTmpRegister(ty, mcv); - return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }); - } else { - const ptr_ty = try pt.singleMutPtrType(ty); - - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp); - const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); - defer for (regs_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const src_reg = regs[0]; - const dst_reg = regs[1]; - const len_reg = regs[2]; - const tmp_reg = regs[3]; - - switch (mcv) { - .stack_offset => |off| try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off }), - .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }), - else => unreachable, - } - - try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset }); - try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size }); - try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg); - } - }, - } -} - -fn genStore(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void { - assert(off_type == Register or off_type == i13); - - const is_imm = (off_type == i13); - - switch (abi_size) { - 1, 2, 4, 8 => { - const tag: Mir.Inst.Tag = switch (abi_size) { - 1 => .stb, - 2 => .sth, - 4 => .stw, - 8 => .stx, - else => unreachable, // unexpected abi size - }; - - _ = try self.addInst(.{ - .tag = tag, - .data = .{ - .arithmetic_3op = .{ - .is_imm = is_imm, - .rd = value_reg, - .rs1 = addr_reg, - .rs2_or_imm = if (is_imm) .{ .imm = off } else .{ .rs2 = off }, - }, - }, - }); - }, - 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), - else => unreachable, - } -} - -fn genStoreASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void { - switch (abi_size) { - 1, 2, 4, 8 => { - const tag: Mir.Inst.Tag = switch (abi_size) { - 1 => .stba, - 2 => .stha, - 4 => .stwa, - 8 => .stxa, - else => unreachable, // unexpected abi size - }; - - _ = try self.addInst(.{ - .tag = tag, - .data = .{ - .mem_asi = .{ - .rd = value_reg, - .rs1 = addr_reg, - .rs2 = off_reg, - .asi = asi, - }, - }, - }); - }, - 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), - else => unreachable, - } -} - -fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { - const pt = self.pt; - const mcv: MCValue = switch (try codegen.genTypedValue( - self.bin_file, - pt, - self.src_loc, - val, - self.target, - )) { - .mcv => |mcv| switch (mcv) { - .none => .none, - .undef => .undef, - .load_got, .load_symbol, .load_direct, .lea_symbol, .lea_direct => unreachable, // TODO - .immediate => |imm| .{ .immediate = imm }, - .memory => |addr| .{ .memory = addr }, - }, - .fail => |msg| { - self.err_msg = msg; - return error.CodegenFail; - }, - }; - return mcv; -} - -fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue { - // Treat each stack item as a "layer" on top of the previous one. - var i: usize = self.branch_stack.items.len; - while (true) { - i -= 1; - if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| { - log.debug("getResolvedInstValue %{f} => {}", .{ inst, mcv }); - assert(mcv != .dead); - return mcv; - } - } -} - -fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const error_type = ty.errorUnionSet(zcu); - const payload_type = ty.errorUnionPayload(zcu); - - if (!error_type.hasRuntimeBits(zcu)) { - return MCValue{ .immediate = 0 }; // always false - } else if (!payload_type.hasRuntimeBits(zcu)) { - if (error_type.abiSize(zcu) <= 8) { - const reg_mcv: MCValue = switch (operand) { - .register => operand, - else => .{ .register = try self.copyToTmpRegister(error_type, operand) }, - }; - - _ = try self.addInst(.{ - .tag = .cmp, - .data = .{ .arithmetic_2op = .{ - .is_imm = true, - .rs1 = reg_mcv.register, - .rs2_or_imm = .{ .imm = 0 }, - } }, - }); - - return MCValue{ .condition_flags = .{ .cond = .{ .icond = .gu }, .ccr = .xcc } }; - } else { - return self.fail("TODO isErr for errors with size > 8", .{}); - } - } else { - return self.fail("TODO isErr for non-empty payloads", .{}); - } -} - -fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue { - // Call isErr, then negate the result. - const is_err_result = try self.isErr(ty, operand); - switch (is_err_result) { - .condition_flags => |op| { - return MCValue{ .condition_flags = .{ .cond = op.cond.negate(), .ccr = op.ccr } }; - }, - .immediate => |imm| { - assert(imm == 0); - return MCValue{ .immediate = 1 }; - }, - else => unreachable, - } -} - -fn isNull(self: *Self, operand: MCValue) !MCValue { - _ = operand; - // Here you can specialize this instruction if it makes sense to, otherwise the default - // will call isNonNull and invert the result. - return self.fail("TODO call isNonNull and invert the result", .{}); -} - -fn isNonNull(self: *Self, operand: MCValue) !MCValue { - // Call isNull, then negate the result. - const is_null_result = try self.isNull(operand); - switch (is_null_result) { - .condition_flags => |op| { - return MCValue{ .condition_flags = .{ .cond = op.cond.negate(), .ccr = op.ccr } }; - }, - .immediate => |imm| { - assert(imm == 0); - return MCValue{ .immediate = 1 }; - }, - else => unreachable, - } -} - -fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb { - try self.ensureProcessDeathCapacity(operand_count + 1); - return BigTomb{ - .function = self, - .inst = inst, - .lbt = self.liveness.iterateBigTomb(inst), - }; -} - -/// Send control flow to `inst`. -fn jump(self: *Self, inst: Mir.Inst.Index) !void { - _ = try self.addInst(.{ - .tag = .bpcc, - .data = .{ - .branch_predict_int = .{ - .cond = .al, - .ccr = .xcc, - .inst = inst, - }, - }, - }); - - // TODO find out a way to fill this delay slot - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); -} - -fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const elem_ty = ptr_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - switch (ptr) { - .none => unreachable, - .undef => unreachable, - .unreach => unreachable, - .dead => unreachable, - .condition_flags, - .condition_register, - .register_with_overflow, - => unreachable, // cannot hold an address - .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }), - .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }), - .register => |addr_reg| { - const addr_reg_lock = self.register_manager.lockReg(addr_reg); - defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg); - - switch (dst_mcv) { - .dead => unreachable, - .undef => unreachable, - .condition_flags => unreachable, - .register => |dst_reg| { - try self.genLoad(dst_reg, addr_reg, i13, 0, elem_size); - }, - .stack_offset => |off| { - if (elem_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null, gp); - const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_reg_lock); - - try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); - try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); - } else { - const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp); - const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs); - defer for (regs_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const src_reg = addr_reg; - const dst_reg = regs[0]; - const len_reg = regs[1]; - const tmp_reg = regs[2]; - - try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off }); - try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size }); - try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg); - } - }, - else => return self.fail("TODO load from register into {}", .{dst_mcv}), - } - }, - .memory, - .stack_offset, - => { - const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr); - try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty); - }, - } -} - -fn minMax( - self: *Self, - tag: Air.Inst.Tag, - lhs: MCValue, - rhs: MCValue, - lhs_ty: Type, - rhs_ty: Type, -) InnerError!MCValue { - const pt = self.pt; - const zcu = pt.zcu; - assert(lhs_ty.eql(rhs_ty, zcu)); - switch (lhs_ty.zigTypeTag(zcu)) { - .float => return self.fail("TODO min/max on floats", .{}), - .vector => return self.fail("TODO min/max on vectors", .{}), - .int => { - const int_info = lhs_ty.intInfo(zcu); - if (int_info.bits <= 64) { - // TODO skip register setting when one of the operands - // is a small (fits in i13) immediate. - const rhs_is_register = rhs == .register; - const rhs_reg = if (rhs_is_register) - rhs.register - else - try self.register_manager.allocReg(null, gp); - const rhs_lock = self.register_manager.lockReg(rhs_reg); - defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); - if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); - - const result_reg = try self.register_manager.allocReg(null, gp); - const result_lock = self.register_manager.lockReg(result_reg); - defer if (result_lock) |reg| self.register_manager.unlockReg(reg); - try self.genSetReg(lhs_ty, result_reg, lhs); - - const cond_choose_rhs: Instruction.ICondition = switch (tag) { - .max => switch (int_info.signedness) { - .signed => Instruction.ICondition.gt, - .unsigned => Instruction.ICondition.gu, - }, - .min => switch (int_info.signedness) { - .signed => Instruction.ICondition.lt, - .unsigned => Instruction.ICondition.cs, - }, - else => unreachable, - }; - - _ = try self.addInst(.{ - .tag = .cmp, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = result_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - }); - - _ = try self.addInst(.{ - .tag = .movcc, - .data = .{ - .conditional_move_int = .{ - .is_imm = false, - .ccr = .xcc, - .cond = .{ .icond = cond_choose_rhs }, - .rd = result_reg, - .rs2_or_imm = .{ .rs2 = rhs_reg }, - }, - }, - }); - - return MCValue{ .register = result_reg }; - } else { - return self.fail("TODO min/max on integers > u64/i64", .{}); - } - }, - else => unreachable, - } -} - -fn parseRegName(name: []const u8) ?Register { - if (@hasDecl(Register, "parseRegName")) { - return Register.parseRegName(name); - } - return std.meta.stringToEnum(Register, name); -} - -fn performReloc(self: *Self, inst: Mir.Inst.Index) !void { - const tag = self.mir_instructions.items(.tag)[inst]; - switch (tag) { - .bpcc => self.mir_instructions.items(.data)[inst].branch_predict_int.inst = @intCast(self.mir_instructions.len), - .bpr => self.mir_instructions.items(.data)[inst].branch_predict_reg.inst = @intCast(self.mir_instructions.len), - else => unreachable, - } -} - -/// Asserts there is already capacity to insert into top branch inst_table. -fn processDeath(self: *Self, inst: Air.Inst.Index) void { - // When editing this function, note that the logic must synchronize with `reuseOperand`. - const prev_value = self.getResolvedInstValue(inst); - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - branch.inst_table.putAssumeCapacity(inst, .dead); - log.debug("%{f} death: {} -> .dead", .{ inst, prev_value }); - switch (prev_value) { - .register => |reg| { - self.register_manager.freeReg(reg); - }, - .register_with_overflow => |rwo| { - self.register_manager.freeReg(rwo.reg); - self.condition_flags_inst = null; - }, - .condition_flags => { - self.condition_flags_inst = null; - }, - else => {}, // TODO process stack allocation death - } -} - -/// Turns stack_offset MCV into a real SPARCv9 stack offset usable for asm. -fn realStackOffset(off: u32) u32 { - return off + - // SPARCv9 %sp points away from the stack by some amount. - abi.stack_bias + - // The first couple bytes of each stack frame is reserved - // for ABI and hardware purposes. - abi.stack_reserved_area; - // Only after that we have the usable stack frame portion. -} - -/// Caller must call `CallMCValues.deinit`. -fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: RegisterView) !CallMCValues { - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const fn_info = zcu.typeToFunc(fn_ty).?; - const cc = fn_info.cc; - var result: CallMCValues = .{ - .args = try self.gpa.alloc(MCValue, fn_info.param_types.len), - // These undefined values must be populated before returning from this function. - .return_value = undefined, - .stack_byte_count = undefined, - .stack_align = undefined, - }; - errdefer self.gpa.free(result.args); - - const ret_ty = fn_ty.fnReturnType(zcu); - - switch (cc) { - .naked => { - assert(result.args.len == 0); - result.return_value = .{ .unreach = {} }; - result.stack_byte_count = 0; - result.stack_align = .@"1"; - return result; - }, - .auto, .sparc64_sysv => { - // SPARC Compliance Definition 2.4.1, Chapter 3 - // Low-Level System Information (64-bit psABI) - Function Calling Sequence - - var next_register: usize = 0; - var next_stack_offset: u32 = 0; - // TODO: this is never assigned, which is a bug, but I don't know how this code works - // well enough to try and fix it. I *think* `next_register += next_stack_offset` is - // supposed to be `next_stack_offset += param_size` in every case where it appears. - _ = &next_stack_offset; - - // The caller puts the argument in %o0-%o5, which becomes %i0-%i5 inside the callee. - const argument_registers = switch (role) { - .caller => abi.c_abi_int_param_regs_caller_view, - .callee => abi.c_abi_int_param_regs_callee_view, - }; - - for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| { - const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu)); - if (param_size <= 8) { - if (next_register < argument_registers.len) { - result_arg.* = .{ .register = argument_registers[next_register] }; - next_register += 1; - } else { - result_arg.* = .{ .stack_offset = next_stack_offset }; - next_register += next_stack_offset; - } - } else if (param_size <= 16) { - if (next_register < argument_registers.len - 1) { - return self.fail("TODO MCValues with 2 registers", .{}); - } else if (next_register < argument_registers.len) { - return self.fail("TODO MCValues split register + stack", .{}); - } else { - result_arg.* = .{ .stack_offset = next_stack_offset }; - next_register += next_stack_offset; - } - } else { - result_arg.* = .{ .stack_offset = next_stack_offset }; - next_register += next_stack_offset; - } - } - - result.stack_byte_count = next_stack_offset; - result.stack_align = .@"16"; - - if (ret_ty.zigTypeTag(zcu) == .noreturn) { - result.return_value = .{ .unreach = {} }; - } else if (!ret_ty.hasRuntimeBits(zcu)) { - result.return_value = .{ .none = {} }; - } else { - const ret_ty_size: u32 = @intCast(ret_ty.abiSize(zcu)); - // The callee puts the return values in %i0-%i3, which becomes %o0-%o3 inside the caller. - if (ret_ty_size <= 8) { - result.return_value = switch (role) { - .caller => .{ .register = abi.c_abi_int_return_regs_caller_view[0] }, - .callee => .{ .register = abi.c_abi_int_return_regs_callee_view[0] }, - }; - } else { - return self.fail("TODO support more return values for sparc64", .{}); - } - } - }, - else => return self.fail("TODO implement function parameters for {} on sparc64", .{cc}), - } - - return result; -} - -fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue { - const pt = self.pt; - const ty = self.typeOf(ref); - - // If the type has no codegen bits, no need to store it. - if (!ty.hasRuntimeBitsIgnoreComptime(pt.zcu)) return .none; - - if (ref.toIndex()) |inst| { - return self.getResolvedInstValue(inst); - } - - return self.genTypedValue((try self.air.value(ref, pt)).?); -} - -fn ret(self: *Self, mcv: MCValue) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ret_ty = self.fn_type.fnReturnType(zcu); - try self.setRegOrMem(ret_ty, self.ret_mcv, mcv); - - // Just add space for a branch instruction, patch this later - const index = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); - - // Reserve space for the delay slot too - // TODO find out a way to fill this - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); - try self.exitlude_jump_relocs.append(self.gpa, index); -} - -fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Air.Liveness.OperandInt, mcv: MCValue) bool { - if (!self.liveness.operandDies(inst, op_index)) - return false; - - switch (mcv) { - .register => |reg| { - // If it's in the registers table, need to associate the register with the - // new instruction. - if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { - if (!self.register_manager.isRegFree(reg)) { - self.register_manager.registers[index] = inst; - } - } - log.debug("%{d} => {} (reused)", .{ inst, reg }); - }, - .stack_offset => |off| { - log.debug("%{d} => stack offset {d} (reused)", .{ inst, off }); - }, - else => return false, - } - - // Prevent the operand deaths processing code from deallocating it. - self.reused_operands.set(op_index); - - // That makes us responsible for doing the rest of the stuff that processDeath would have done. - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - branch.inst_table.putAssumeCapacity(operand.toIndex().?, .dead); - - return true; -} - -/// Sets the value without any modifications to register allocation metadata or stack allocation metadata. -fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void { - switch (loc) { - .none => return, - .register => |reg| return self.genSetReg(ty, reg, val), - .stack_offset => |off| return self.genSetStack(ty, off, val), - .memory => { - return self.fail("TODO implement setRegOrMem for memory", .{}); - }, - else => unreachable, - } -} - -/// Save the current instruction stored in the condition flags if -/// occupied -fn spillConditionFlagsIfOccupied(self: *Self) !void { - if (self.condition_flags_inst) |inst_to_save| { - const mcv = self.getResolvedInstValue(inst_to_save); - const new_mcv = switch (mcv) { - .condition_flags => try self.allocRegOrMem(inst_to_save, true), - .register_with_overflow => try self.allocRegOrMem(inst_to_save, false), - else => unreachable, // mcv doesn't occupy the compare flags - }; - - try self.setRegOrMem(self.typeOfIndex(inst_to_save), new_mcv, mcv); - log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv }); - - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - try branch.inst_table.put(self.gpa, inst_to_save, new_mcv); - - self.condition_flags_inst = null; - - // TODO consolidate with register manager and spillInstruction - // this call should really belong in the register manager! - switch (mcv) { - .register_with_overflow => |rwo| self.register_manager.freeReg(rwo.reg), - else => {}, - } - } -} - -pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { - const stack_mcv = try self.allocRegOrMem(inst, false); - log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv }); - const reg_mcv = self.getResolvedInstValue(inst); - assert(reg == reg_mcv.register); - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - try branch.inst_table.put(self.gpa, inst, stack_mcv); - try self.genSetStack(self.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv); -} - -fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { - const pt = self.pt; - const abi_size = value_ty.abiSize(pt.zcu); - - switch (ptr) { - .none => unreachable, - .undef => unreachable, - .unreach => unreachable, - .dead => unreachable, - .condition_flags, - .condition_register, - .register_with_overflow, - => unreachable, // cannot hold an address - .immediate => |imm| { - try self.setRegOrMem(value_ty, .{ .memory = imm }, value); - }, - .ptr_stack_offset => |off| { - try self.genSetStack(value_ty, off, value); - }, - .register => |addr_reg| { - const addr_reg_lock = self.register_manager.lockReg(addr_reg); - defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg); - - switch (value) { - .register => |value_reg| { - try self.genStore(value_reg, addr_reg, i13, 0, abi_size); - }, - else => { - return self.fail("TODO implement copying of memory", .{}); - }, - } - }, - .memory, - .stack_offset, - => { - const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr); - try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty); - }, - } -} - -fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue { - return if (self.liveness.isUnused(inst)) .dead else result: { - const pt = self.pt; - const zcu = pt.zcu; - const mcv = try self.resolveInst(operand); - const ptr_ty = self.typeOf(operand); - const struct_ty = ptr_ty.childType(zcu); - const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu)); - switch (mcv) { - .ptr_stack_offset => |off| { - break :result MCValue{ .ptr_stack_offset = off - struct_field_offset }; - }, - else => { - const offset_reg = try self.copyToTmpRegister(ptr_ty, .{ - .immediate = struct_field_offset, - }); - const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); - defer self.register_manager.unlockReg(offset_reg_lock); - - const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv); - const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_reg_lock); - - const dest = try self.binOp( - .add, - .{ .register = addr_reg }, - .{ .register = offset_reg }, - Type.usize, - Type.usize, - null, - ); - - break :result dest; - }, - } - }; -} - -fn trunc( - self: *Self, - maybe_inst: ?Air.Inst.Index, - operand: MCValue, - operand_ty: Type, - dest_ty: Type, -) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const info_a = operand_ty.intInfo(zcu); - const info_b = dest_ty.intInfo(zcu); - - if (info_b.bits <= 64) { - const operand_reg = switch (operand) { - .register => |r| r, - else => operand_reg: { - if (info_a.bits <= 64) { - const reg = try self.copyToTmpRegister(operand_ty, operand); - break :operand_reg reg; - } else { - return self.fail("TODO load least significant word into register", .{}); - } - }, - }; - const lock = self.register_manager.lockReg(operand_reg); - defer if (lock) |reg| self.register_manager.unlockReg(reg); - - const dest_reg = if (maybe_inst) |inst| blk: { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { - break :blk operand_reg; - } else { - const reg = try self.register_manager.allocReg(inst, gp); - break :blk reg; - } - } else blk: { - const reg = try self.register_manager.allocReg(null, gp); - break :blk reg; - }; - - try self.truncRegister(operand_reg, dest_reg, info_b.signedness, info_b.bits); - - return MCValue{ .register = dest_reg }; - } else { - return self.fail("TODO: truncate to ints > 64 bits", .{}); - } -} - -fn truncRegister( - self: *Self, - operand_reg: Register, - dest_reg: Register, - int_signedness: std.builtin.Signedness, - int_bits: u16, -) !void { - switch (int_bits) { - 1...31, 33...63 => { - _ = try self.addInst(.{ - .tag = .sllx, - .data = .{ - .shift = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = operand_reg, - .rs2_or_imm = .{ .imm = @as(u6, @intCast(64 - int_bits)) }, - }, - }, - }); - _ = try self.addInst(.{ - .tag = switch (int_signedness) { - .signed => .srax, - .unsigned => .srlx, - }, - .data = .{ - .shift = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = dest_reg, - .rs2_or_imm = .{ .imm = @as(u6, @intCast(int_bits)) }, - }, - }, - }); - }, - 32 => { - _ = try self.addInst(.{ - .tag = switch (int_signedness) { - .signed => .sra, - .unsigned => .srl, - }, - .data = .{ - .shift = .{ - .is_imm = true, - .rd = dest_reg, - .rs1 = operand_reg, - .rs2_or_imm = .{ .imm = 0 }, - }, - }, - }); - }, - 64 => { - if (dest_reg == operand_reg) - return; // Copy register to itself; nothing to do. - _ = try self.addInst(.{ - .tag = .mov, - .data = .{ - .arithmetic_2op = .{ - .is_imm = false, - .rs1 = dest_reg, - .rs2_or_imm = .{ .rs2 = operand_reg }, - }, - }, - }); - }, - else => unreachable, - } -} - -/// TODO support scope overrides. Also note this logic is duplicated with `Zcu.wantSafety`. -fn wantSafety(self: *Self) bool { - return switch (self.bin_file.comp.root_mod.optimize_mode) { - .Debug => true, - .ReleaseSafe => true, - .ReleaseFast => false, - .ReleaseSmall => false, - }; -} - -fn typeOf(self: *Self, inst: Air.Inst.Ref) Type { - return self.air.typeOf(inst, &self.pt.zcu.intern_pool); -} - -fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type { - return self.air.typeOfIndex(inst, &self.pt.zcu.intern_pool); -} diff --git a/src/arch/sparc64/Emit.zig b/src/arch/sparc64/Emit.zig deleted file mode 100644 index 26282b09ab4f5c831cea32e9c9dbf8d6e0b2256a..0000000000000000000000000000000000000000 --- a/src/arch/sparc64/Emit.zig +++ /dev/null @@ -1,682 +0,0 @@ -//! This file contains the functionality for lowering SPARCv9 MIR into -//! machine code - -const std = @import("std"); -const Endian = std.builtin.Endian; -const assert = std.debug.assert; -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); -const ErrorMsg = Zcu.ErrorMsg; -const log = std.log.scoped(.sparcv9_emit); - -const Emit = @This(); -const Mir = @import("Mir.zig"); -const bits = @import("bits.zig"); -const Instruction = bits.Instruction; -const Register = bits.Register; - -mir: Mir, -bin_file: *link.File, -debug_output: link.File.DebugInfoOutput, -target: *const std.Target, -err_msg: ?*ErrorMsg = null, -src_loc: Zcu.LazySrcLoc, -w: *std.Io.Writer, - -prev_di_line: u32, -prev_di_column: u32, -/// Relative to the beginning of `code`. -prev_di_pc: usize, - -/// The branch type of every branch -branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .empty, -/// For every forward branch, maps the target instruction to a list of -/// branches which branch to this target instruction -branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .empty, -/// For backward branches: stores the code offset of the target -/// instruction -/// -/// For forward branches: stores the code offset of the branch -/// instruction -code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty, - -const InnerError = std.Io.Writer.Error || error{ - OutOfMemory, - EmitFail, -}; - -const BranchType = enum { - bpcc, - bpr, - fn default(tag: Mir.Inst.Tag) BranchType { - return switch (tag) { - .bpcc => .bpcc, - .bpr => .bpr, - else => unreachable, - }; - } -}; - -pub fn emitMir( - emit: *Emit, -) InnerError!void { - const mir_tags = emit.mir.instructions.items(.tag); - - // Convert absolute addresses into offsets and - // find smallest lowerings for branch instructions - try emit.lowerBranches(); - - // Emit machine code - for (mir_tags, 0..) |tag, index| { - const inst = @as(u32, @intCast(index)); - switch (tag) { - .dbg_line => try emit.mirDbgLine(inst), - .dbg_prologue_end => try emit.mirDebugPrologueEnd(), - .dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(), - - .add => try emit.mirArithmetic3Op(inst), - .addcc => try emit.mirArithmetic3Op(inst), - - .bpr => try emit.mirConditionalBranch(inst), - .bpcc => try emit.mirConditionalBranch(inst), - - .call => @panic("TODO implement sparc64 call"), - - .jmpl => try emit.mirArithmetic3Op(inst), - - .ldub => try emit.mirArithmetic3Op(inst), - .lduh => try emit.mirArithmetic3Op(inst), - .lduw => try emit.mirArithmetic3Op(inst), - .ldx => try emit.mirArithmetic3Op(inst), - - .lduba => try emit.mirMemASI(inst), - .lduha => try emit.mirMemASI(inst), - .lduwa => try emit.mirMemASI(inst), - .ldxa => try emit.mirMemASI(inst), - - .@"and" => try emit.mirArithmetic3Op(inst), - .@"or" => try emit.mirArithmetic3Op(inst), - .xor => try emit.mirArithmetic3Op(inst), - .xnor => try emit.mirArithmetic3Op(inst), - - .membar => try emit.mirMembar(inst), - - .movcc => try emit.mirConditionalMove(inst), - - .movr => try emit.mirConditionalMove(inst), - - .mulx => try emit.mirArithmetic3Op(inst), - .sdivx => try emit.mirArithmetic3Op(inst), - .udivx => try emit.mirArithmetic3Op(inst), - - .nop => try emit.mirNop(), - - .@"return" => try emit.mirArithmetic2Op(inst), - - .save => try emit.mirArithmetic3Op(inst), - .restore => try emit.mirArithmetic3Op(inst), - - .sethi => try emit.mirSethi(inst), - - .sll => try emit.mirShift(inst), - .srl => try emit.mirShift(inst), - .sra => try emit.mirShift(inst), - .sllx => try emit.mirShift(inst), - .srlx => try emit.mirShift(inst), - .srax => try emit.mirShift(inst), - - .stb => try emit.mirArithmetic3Op(inst), - .sth => try emit.mirArithmetic3Op(inst), - .stw => try emit.mirArithmetic3Op(inst), - .stx => try emit.mirArithmetic3Op(inst), - - .stba => try emit.mirMemASI(inst), - .stha => try emit.mirMemASI(inst), - .stwa => try emit.mirMemASI(inst), - .stxa => try emit.mirMemASI(inst), - - .sub => try emit.mirArithmetic3Op(inst), - .subcc => try emit.mirArithmetic3Op(inst), - - .tcc => try emit.mirTrap(inst), - - .cmp => try emit.mirArithmetic2Op(inst), - - .mov => try emit.mirArithmetic2Op(inst), - - .not => try emit.mirArithmetic2Op(inst), - } - } -} - -pub fn deinit(emit: *Emit) void { - const comp = emit.bin_file.comp; - const gpa = comp.gpa; - var iter = emit.branch_forward_origins.valueIterator(); - while (iter.next()) |origin_list| { - origin_list.deinit(gpa); - } - - emit.branch_types.deinit(gpa); - emit.branch_forward_origins.deinit(gpa); - emit.code_offset_mapping.deinit(gpa); - emit.* = undefined; -} - -fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column; - - switch (tag) { - .dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column), - else => unreachable, - } -} - -fn mirDebugPrologueEnd(emit: *Emit) !void { - switch (emit.debug_output) { - .dwarf => |dbg_out| { - try dbg_out.setPrologueEnd(); - try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); - }, - .none => {}, - } -} - -fn mirDebugEpilogueBegin(emit: *Emit) !void { - switch (emit.debug_output) { - .dwarf => |dbg_out| { - try dbg_out.setEpilogueBegin(); - try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); - }, - .none => {}, - } -} - -fn mirArithmetic2Op(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const data = emit.mir.instructions.items(.data)[inst].arithmetic_2op; - - const rs1 = data.rs1; - - if (data.is_imm) { - const imm = data.rs2_or_imm.imm; - switch (tag) { - .@"return" => try emit.writeInstruction(Instruction.@"return"(i13, rs1, imm)), - .cmp => try emit.writeInstruction(Instruction.subcc(i13, rs1, imm, .g0)), - .mov => try emit.writeInstruction(Instruction.@"or"(i13, .g0, imm, rs1)), - .not => try emit.writeInstruction(Instruction.xnor(i13, .g0, imm, rs1)), - else => unreachable, - } - } else { - const rs2 = data.rs2_or_imm.rs2; - switch (tag) { - .@"return" => try emit.writeInstruction(Instruction.@"return"(Register, rs1, rs2)), - .cmp => try emit.writeInstruction(Instruction.subcc(Register, rs1, rs2, .g0)), - .mov => try emit.writeInstruction(Instruction.@"or"(Register, .g0, rs2, rs1)), - .not => try emit.writeInstruction(Instruction.xnor(Register, rs2, .g0, rs1)), - else => unreachable, - } - } -} - -fn mirArithmetic3Op(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const data = emit.mir.instructions.items(.data)[inst].arithmetic_3op; - - const rd = data.rd; - const rs1 = data.rs1; - - if (data.is_imm) { - const imm = data.rs2_or_imm.imm; - switch (tag) { - .add => try emit.writeInstruction(Instruction.add(i13, rs1, imm, rd)), - .addcc => try emit.writeInstruction(Instruction.addcc(i13, rs1, imm, rd)), - .jmpl => try emit.writeInstruction(Instruction.jmpl(i13, rs1, imm, rd)), - .ldub => try emit.writeInstruction(Instruction.ldub(i13, rs1, imm, rd)), - .lduh => try emit.writeInstruction(Instruction.lduh(i13, rs1, imm, rd)), - .lduw => try emit.writeInstruction(Instruction.lduw(i13, rs1, imm, rd)), - .ldx => try emit.writeInstruction(Instruction.ldx(i13, rs1, imm, rd)), - .@"and" => try emit.writeInstruction(Instruction.@"and"(i13, rs1, imm, rd)), - .@"or" => try emit.writeInstruction(Instruction.@"or"(i13, rs1, imm, rd)), - .xor => try emit.writeInstruction(Instruction.xor(i13, rs1, imm, rd)), - .xnor => try emit.writeInstruction(Instruction.xnor(i13, rs1, imm, rd)), - .mulx => try emit.writeInstruction(Instruction.mulx(i13, rs1, imm, rd)), - .sdivx => try emit.writeInstruction(Instruction.sdivx(i13, rs1, imm, rd)), - .udivx => try emit.writeInstruction(Instruction.udivx(i13, rs1, imm, rd)), - .save => try emit.writeInstruction(Instruction.save(i13, rs1, imm, rd)), - .restore => try emit.writeInstruction(Instruction.restore(i13, rs1, imm, rd)), - .stb => try emit.writeInstruction(Instruction.stb(i13, rs1, imm, rd)), - .sth => try emit.writeInstruction(Instruction.sth(i13, rs1, imm, rd)), - .stw => try emit.writeInstruction(Instruction.stw(i13, rs1, imm, rd)), - .stx => try emit.writeInstruction(Instruction.stx(i13, rs1, imm, rd)), - .sub => try emit.writeInstruction(Instruction.sub(i13, rs1, imm, rd)), - .subcc => try emit.writeInstruction(Instruction.subcc(i13, rs1, imm, rd)), - else => unreachable, - } - } else { - const rs2 = data.rs2_or_imm.rs2; - switch (tag) { - .add => try emit.writeInstruction(Instruction.add(Register, rs1, rs2, rd)), - .addcc => try emit.writeInstruction(Instruction.addcc(Register, rs1, rs2, rd)), - .jmpl => try emit.writeInstruction(Instruction.jmpl(Register, rs1, rs2, rd)), - .ldub => try emit.writeInstruction(Instruction.ldub(Register, rs1, rs2, rd)), - .lduh => try emit.writeInstruction(Instruction.lduh(Register, rs1, rs2, rd)), - .lduw => try emit.writeInstruction(Instruction.lduw(Register, rs1, rs2, rd)), - .ldx => try emit.writeInstruction(Instruction.ldx(Register, rs1, rs2, rd)), - .@"and" => try emit.writeInstruction(Instruction.@"and"(Register, rs1, rs2, rd)), - .@"or" => try emit.writeInstruction(Instruction.@"or"(Register, rs1, rs2, rd)), - .xor => try emit.writeInstruction(Instruction.xor(Register, rs1, rs2, rd)), - .xnor => try emit.writeInstruction(Instruction.xnor(Register, rs1, rs2, rd)), - .mulx => try emit.writeInstruction(Instruction.mulx(Register, rs1, rs2, rd)), - .sdivx => try emit.writeInstruction(Instruction.sdivx(Register, rs1, rs2, rd)), - .udivx => try emit.writeInstruction(Instruction.udivx(Register, rs1, rs2, rd)), - .save => try emit.writeInstruction(Instruction.save(Register, rs1, rs2, rd)), - .restore => try emit.writeInstruction(Instruction.restore(Register, rs1, rs2, rd)), - .stb => try emit.writeInstruction(Instruction.stb(Register, rs1, rs2, rd)), - .sth => try emit.writeInstruction(Instruction.sth(Register, rs1, rs2, rd)), - .stw => try emit.writeInstruction(Instruction.stw(Register, rs1, rs2, rd)), - .stx => try emit.writeInstruction(Instruction.stx(Register, rs1, rs2, rd)), - .sub => try emit.writeInstruction(Instruction.sub(Register, rs1, rs2, rd)), - .subcc => try emit.writeInstruction(Instruction.subcc(Register, rs1, rs2, rd)), - else => unreachable, - } - } -} - -fn mirConditionalBranch(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const branch_type = emit.branch_types.get(inst).?; - - switch (branch_type) { - .bpcc => switch (tag) { - .bpcc => { - const branch_predict_int = emit.mir.instructions.items(.data)[inst].branch_predict_int; - const offset = @as(i64, @intCast(emit.code_offset_mapping.get(branch_predict_int.inst).?)) - @as(i64, @intCast(emit.w.end)); - log.debug("mirConditionalBranch: {} offset={}", .{ inst, offset }); - - try emit.writeInstruction( - Instruction.bpcc( - branch_predict_int.cond, - branch_predict_int.annul, - branch_predict_int.pt, - branch_predict_int.ccr, - @as(i21, @intCast(offset)), - ), - ); - }, - else => unreachable, - }, - .bpr => switch (tag) { - .bpr => { - const branch_predict_reg = emit.mir.instructions.items(.data)[inst].branch_predict_reg; - const offset = @as(i64, @intCast(emit.code_offset_mapping.get(branch_predict_reg.inst).?)) - @as(i64, @intCast(emit.w.end)); - log.debug("mirConditionalBranch: {} offset={}", .{ inst, offset }); - - try emit.writeInstruction( - Instruction.bpr( - branch_predict_reg.cond, - branch_predict_reg.annul, - branch_predict_reg.pt, - branch_predict_reg.rs1, - @as(i18, @intCast(offset)), - ), - ); - }, - else => unreachable, - }, - } -} - -fn mirConditionalMove(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - - switch (tag) { - .movcc => { - const data = emit.mir.instructions.items(.data)[inst].conditional_move_int; - if (data.is_imm) { - try emit.writeInstruction(Instruction.movcc( - i11, - data.cond, - data.ccr, - data.rs2_or_imm.imm, - data.rd, - )); - } else { - try emit.writeInstruction(Instruction.movcc( - Register, - data.cond, - data.ccr, - data.rs2_or_imm.rs2, - data.rd, - )); - } - }, - .movr => { - const data = emit.mir.instructions.items(.data)[inst].conditional_move_reg; - if (data.is_imm) { - try emit.writeInstruction(Instruction.movr( - i10, - data.cond, - data.rs1, - data.rs2_or_imm.imm, - data.rd, - )); - } else { - try emit.writeInstruction(Instruction.movr( - Register, - data.cond, - data.rs1, - data.rs2_or_imm.rs2, - data.rd, - )); - } - }, - else => unreachable, - } -} - -fn mirMemASI(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const data = emit.mir.instructions.items(.data)[inst].mem_asi; - - const rd = data.rd; - const rs1 = data.rs1; - const rs2 = data.rs2; - const asi = data.asi; - - switch (tag) { - .lduba => try emit.writeInstruction(Instruction.lduba(rs1, rs2, asi, rd)), - .lduha => try emit.writeInstruction(Instruction.lduha(rs1, rs2, asi, rd)), - .lduwa => try emit.writeInstruction(Instruction.lduwa(rs1, rs2, asi, rd)), - .ldxa => try emit.writeInstruction(Instruction.ldxa(rs1, rs2, asi, rd)), - - .stba => try emit.writeInstruction(Instruction.stba(rs1, rs2, asi, rd)), - .stha => try emit.writeInstruction(Instruction.stha(rs1, rs2, asi, rd)), - .stwa => try emit.writeInstruction(Instruction.stwa(rs1, rs2, asi, rd)), - .stxa => try emit.writeInstruction(Instruction.stxa(rs1, rs2, asi, rd)), - else => unreachable, - } -} - -fn mirMembar(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const mask = emit.mir.instructions.items(.data)[inst].membar_mask; - assert(tag == .membar); - - try emit.writeInstruction(Instruction.membar( - mask.cmask, - mask.mmask, - )); -} - -fn mirNop(emit: *Emit) !void { - try emit.writeInstruction(Instruction.nop()); -} - -fn mirSethi(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const data = emit.mir.instructions.items(.data)[inst].sethi; - - const imm = data.imm; - const rd = data.rd; - - assert(tag == .sethi); - try emit.writeInstruction(Instruction.sethi(imm, rd)); -} - -fn mirShift(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const data = emit.mir.instructions.items(.data)[inst].shift; - - const rd = data.rd; - const rs1 = data.rs1; - - if (data.is_imm) { - const imm = data.rs2_or_imm.imm; - switch (tag) { - .sll => try emit.writeInstruction(Instruction.sll(u5, rs1, @as(u5, @truncate(imm)), rd)), - .srl => try emit.writeInstruction(Instruction.srl(u5, rs1, @as(u5, @truncate(imm)), rd)), - .sra => try emit.writeInstruction(Instruction.sra(u5, rs1, @as(u5, @truncate(imm)), rd)), - .sllx => try emit.writeInstruction(Instruction.sllx(u6, rs1, imm, rd)), - .srlx => try emit.writeInstruction(Instruction.srlx(u6, rs1, imm, rd)), - .srax => try emit.writeInstruction(Instruction.srax(u6, rs1, imm, rd)), - else => unreachable, - } - } else { - const rs2 = data.rs2_or_imm.rs2; - switch (tag) { - .sll => try emit.writeInstruction(Instruction.sll(Register, rs1, rs2, rd)), - .srl => try emit.writeInstruction(Instruction.srl(Register, rs1, rs2, rd)), - .sra => try emit.writeInstruction(Instruction.sra(Register, rs1, rs2, rd)), - .sllx => try emit.writeInstruction(Instruction.sllx(Register, rs1, rs2, rd)), - .srlx => try emit.writeInstruction(Instruction.srlx(Register, rs1, rs2, rd)), - .srax => try emit.writeInstruction(Instruction.srax(Register, rs1, rs2, rd)), - else => unreachable, - } - } -} - -fn mirTrap(emit: *Emit, inst: Mir.Inst.Index) !void { - const tag = emit.mir.instructions.items(.tag)[inst]; - const data = emit.mir.instructions.items(.data)[inst].trap; - - const cond = data.cond; - const ccr = data.ccr; - const rs1 = data.rs1; - - if (data.is_imm) { - const imm = data.rs2_or_imm.imm; - switch (tag) { - .tcc => try emit.writeInstruction(Instruction.trap(u7, cond, ccr, rs1, imm)), - else => unreachable, - } - } else { - const rs2 = data.rs2_or_imm.rs2; - switch (tag) { - .tcc => try emit.writeInstruction(Instruction.trap(Register, cond, ccr, rs1, rs2)), - else => unreachable, - } - } -} - -// Common helper functions - -fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index { - const tag = emit.mir.instructions.items(.tag)[inst]; - - switch (tag) { - .bpcc => return emit.mir.instructions.items(.data)[inst].branch_predict_int.inst, - .bpr => return emit.mir.instructions.items(.data)[inst].branch_predict_reg.inst, - else => unreachable, - } -} - -fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) !void { - const delta_line = @as(i32, @intCast(line)) - @as(i32, @intCast(emit.prev_di_line)); - const delta_pc: usize = emit.w.end - emit.prev_di_pc; - switch (emit.debug_output) { - .dwarf => |dbg_out| { - try dbg_out.advancePCAndLine(delta_line, delta_pc); - emit.prev_di_line = line; - emit.prev_di_column = column; - emit.prev_di_pc = emit.w.end; - }, - else => {}, - } -} - -fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { - @branchHint(.cold); - assert(emit.err_msg == null); - const comp = emit.bin_file.comp; - const gpa = comp.gpa; - emit.err_msg = try ErrorMsg.create(gpa, emit.src_loc, format, args); - return error.EmitFail; -} - -fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize { - const tag = emit.mir.instructions.items(.tag)[inst]; - - switch (tag) { - .dbg_line, - .dbg_epilogue_begin, - .dbg_prologue_end, - => return 0, - // Currently Mir instructions always map to single machine instruction. - else => return 4, - } -} - -fn isBranch(tag: Mir.Inst.Tag) bool { - return switch (tag) { - .bpcc => true, - .bpr => true, - else => false, - }; -} - -fn lowerBranches(emit: *Emit) !void { - const comp = emit.bin_file.comp; - const gpa = comp.gpa; - const mir_tags = emit.mir.instructions.items(.tag); - - // First pass: Note down all branches and their target - // instructions, i.e. populate branch_types, - // branch_forward_origins, and code_offset_mapping - // - // TODO optimization opportunity: do this in codegen while - // generating MIR - for (mir_tags, 0..) |tag, index| { - const inst = @as(u32, @intCast(index)); - if (isBranch(tag)) { - const target_inst = emit.branchTarget(inst); - - // Remember this branch instruction - try emit.branch_types.put(gpa, inst, BranchType.default(tag)); - - // Forward branches require some extra stuff: We only - // know their offset once we arrive at the target - // instruction. Therefore, we need to be able to - // access the branch instruction when we visit the - // target instruction in order to manipulate its type - // etc. - if (target_inst > inst) { - // Remember the branch instruction index - try emit.code_offset_mapping.put(gpa, inst, 0); - - if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| { - try origin_list.append(gpa, inst); - } else { - var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty; - try origin_list.append(gpa, inst); - try emit.branch_forward_origins.put(gpa, target_inst, origin_list); - } - } - - // Remember the target instruction index so that we - // update the real code offset in all future passes - // - // putNoClobber may not be used as the put operation - // may clobber the entry when multiple branches branch - // to the same target instruction - try emit.code_offset_mapping.put(gpa, target_inst, 0); - } - } - - // Further passes: Until all branches are lowered, interate - // through all instructions and calculate new offsets and - // potentially new branch types - var all_branches_lowered = false; - while (!all_branches_lowered) { - all_branches_lowered = true; - var current_code_offset: usize = 0; - - for (mir_tags, 0..) |tag, index| { - const inst = @as(u32, @intCast(index)); - - // If this instruction contained in the code offset - // mapping (when it is a target of a branch or if it is a - // forward branch), update the code offset - if (emit.code_offset_mapping.getPtr(inst)) |offset| { - offset.* = current_code_offset; - } - - // If this instruction is a backward branch, calculate the - // offset, which may potentially update the branch type - if (isBranch(tag)) { - const target_inst = emit.branchTarget(inst); - if (target_inst < inst) { - const target_offset = emit.code_offset_mapping.get(target_inst).?; - const offset = @as(i64, @intCast(target_offset)) - @as(i64, @intCast(current_code_offset)); - const branch_type = emit.branch_types.getPtr(inst).?; - const optimal_branch_type = try emit.optimalBranchType(tag, offset); - if (branch_type.* != optimal_branch_type) { - branch_type.* = optimal_branch_type; - all_branches_lowered = false; - } - - log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset }); - } - } - - // If this instruction is the target of one or more - // forward branches, calculate the offset, which may - // potentially update the branch type - if (emit.branch_forward_origins.get(inst)) |origin_list| { - for (origin_list.items) |forward_branch_inst| { - const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst]; - const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?; - const offset = @as(i64, @intCast(current_code_offset)) - @as(i64, @intCast(forward_branch_inst_offset)); - const branch_type = emit.branch_types.getPtr(forward_branch_inst).?; - const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset); - if (branch_type.* != optimal_branch_type) { - branch_type.* = optimal_branch_type; - all_branches_lowered = false; - } - - log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset }); - } - } - - // Increment code offset - current_code_offset += emit.instructionSize(inst); - } - } -} - -fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType { - assert(offset & 0b11 == 0); - - switch (tag) { - // TODO use the following strategy to implement long branches: - // - Negate the conditional and target of the original instruction; - // - In the space immediately after the branch, load - // the address of the original target, preferrably in - // a PC-relative way, into %o7; and - // - jmpl %o7 + %g0, %g0 - - .bpcc => { - if (std.math.cast(i21, offset)) |_| { - return BranchType.bpcc; - } else { - return emit.fail("TODO support BPcc branches larger than +-1 MiB", .{}); - } - }, - .bpr => { - if (std.math.cast(i18, offset)) |_| { - return BranchType.bpr; - } else { - return emit.fail("TODO support BPr branches larger than +-128 KiB", .{}); - } - }, - else => unreachable, - } -} - -fn writeInstruction(emit: *Emit, instruction: Instruction) !void { - // SPARCv9 instructions are always arranged in BE regardless of the - // endianness mode the CPU is running in (Section 3.1 of the ISA specification). - // This is to ease porting in case someone wants to do a LE SPARCv9 backend. - try emit.w.writeInt(u32, instruction.toU32(), .big); -} diff --git a/src/arch/sparc64/Mir.zig b/src/arch/sparc64/Mir.zig deleted file mode 100644 index 41d135eb831fefe51afd3fde7925596fce9d441b..0000000000000000000000000000000000000000 --- a/src/arch/sparc64/Mir.zig +++ /dev/null @@ -1,428 +0,0 @@ -//! Machine Intermediate Representation. -//! This data is produced by SPARCv9 Codegen or SPARCv9 assembly parsing -//! These instructions have a 1:1 correspondence with machine code instructions -//! for the target. MIR can be lowered to source-annotated textual assembly code -//! instructions, or it can be lowered to machine code. -//! The main purpose of MIR is to postpone the assignment of offsets until Isel, -//! so that, for example, the smaller encodings of jump instructions can be used. - -const std = @import("std"); -const builtin = @import("builtin"); -const assert = std.debug.assert; - -const Mir = @This(); -const bits = @import("bits.zig"); -const InternPool = @import("../../InternPool.zig"); -const Emit = @import("Emit.zig"); -const codegen = @import("../../codegen.zig"); -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); - -const Instruction = bits.Instruction; -const ASI = bits.Instruction.ASI; -const Register = bits.Register; - -instructions: std.MultiArrayList(Inst).Slice, - -/// The meaning of this data is determined by `Inst.Tag` value. -extra: []const u32, - -pub const Inst = struct { - tag: Tag, - /// The meaning of this depends on `tag`. - data: Data, - - pub const Tag = enum(u16) { - /// Pseudo-instruction: End of prologue - dbg_prologue_end, - /// Pseudo-instruction: Beginning of epilogue - dbg_epilogue_begin, - /// Pseudo-instruction: Update debug line - dbg_line, - - // All the real instructions are ordered by their section number - // in The SPARC Architecture Manual, Version 9. - - /// A.2 Add - /// This uses the arithmetic_3op field. - // TODO add other operations. - add, - addcc, - - /// A.3 Branch on Integer Register with Prediction (BPr) - /// This uses the branch_predict_reg field. - bpr, - - /// A.7 Branch on Integer Condition Codes with Prediction (BPcc) - /// This uses the branch_predict_int field. - bpcc, - - /// A.8 Call and Link - /// This uses the branch_link field. - call, - - /// A.24 Jump and Link - /// This uses the arithmetic_3op field. - jmpl, - - /// A.27 Load Integer - /// This uses the arithmetic_3op field. - /// Note that the ldd variant of this instruction is deprecated, so do not emit - /// it unless specifically requested (e.g. by inline assembly). - // TODO add other operations. - ldub, - lduh, - lduw, - ldx, - - /// A.28 Load Integer from Alternate Space - /// This uses the mem_asi field. - /// Note that the ldda variant of this instruction is deprecated, so do not emit - /// it unless specifically requested (e.g. by inline assembly). - // TODO add other operations. - lduba, - lduha, - lduwa, - ldxa, - - /// A.31 Logical Operations - /// This uses the arithmetic_3op field. - // TODO add other operations. - @"and", - @"or", - xor, - xnor, - - /// A.32 Memory Barrier - /// This uses the membar_mask field. - membar, - - /// A.35 Move Integer Register on Condition (MOVcc) - /// This uses the conditional_move_int field. - movcc, - - /// A.36 Move Integer Register on Register Condition (MOVr) - /// This uses the conditional_move_reg field. - movr, - - /// A.37 Multiply and Divide (64-bit) - /// This uses the arithmetic_3op field. - mulx, - sdivx, - udivx, - - /// A.40 No Operation - /// This uses the nop field. - nop, - - /// A.45 RETURN - /// This uses the arithmetic_2op field. - @"return", - - /// A.46 SAVE and RESTORE - /// This uses the arithmetic_3op field. - save, - restore, - - /// A.48 SETHI - /// This uses the sethi field. - sethi, - - /// A.49 Shift - /// This uses the shift field. - sll, - srl, - sra, - sllx, - srlx, - srax, - - /// A.54 Store Integer - /// This uses the arithmetic_3op field. - /// Note that the std variant of this instruction is deprecated, so do not emit - /// it unless specifically requested (e.g. by inline assembly). - // TODO add other operations. - stb, - sth, - stw, - stx, - - /// A.55 Store Integer into Alternate Space - /// This uses the mem_asi field. - /// Note that the stda variant of this instruction is deprecated, so do not emit - /// it unless specifically requested (e.g. by inline assembly). - // TODO add other operations. - stba, - stha, - stwa, - stxa, - - /// A.56 Subtract - /// This uses the arithmetic_3op field. - // TODO add other operations. - sub, - subcc, - - /// A.61 Trap on Integer Condition Codes (Tcc) - /// This uses the trap field. - tcc, - - // SPARCv9 synthetic instructions - // Note that the instructions that is added here are only those that - // will simplify backend development. Synthetic instructions that is - // only used to provide syntactic sugar in, e.g. inline assembly should - // be deconstructed inside the parser instead. - // See also: G.3 Synthetic Instructions - // TODO add more synthetic instructions - - /// Comparison - /// This uses the arithmetic_2op field. - cmp, // cmp rs1, rs2/imm -> subcc rs1, rs2/imm, %g0 - - /// Copy register/immediate contents to another register - /// This uses the arithmetic_2op field, with rs1 - /// being the *destination* register. - // TODO is it okay to abuse rs1 in this way? - mov, // mov rs2/imm, rs1 -> or %g0, rs2/imm, rs1 - - /// Bitwise negation - /// This uses the arithmetic_2op field, with rs1 - /// being the *destination* register. - // TODO is it okay to abuse rs1 in this way? - // not rs2, rs1 -> xnor rs2, %g0, rs1 - // not imm, rs1 -> xnor %g0, imm, rs1 - not, - }; - - /// The position of an MIR instruction within the `Mir` instructions array. - pub const Index = u32; - - /// All instructions have a 8-byte payload, which is contained within - /// this union. `Tag` determines which union field is active, as well as - /// how to interpret the data within. - // TODO this is a quick-n-dirty solution that needs to be cleaned up. - pub const Data = union { - /// Debug info: line and column - /// - /// Used by e.g. dbg_line - dbg_line_column: struct { - line: u32, - column: u32, - }, - - /// Two operand arithmetic. - /// if is_imm true then it uses the imm field of rs2_or_imm, - /// otherwise it uses rs2 field. - /// - /// Used by e.g. return - arithmetic_2op: struct { - is_imm: bool, - rs1: Register, - rs2_or_imm: union { - rs2: Register, - imm: i13, - }, - }, - - /// Three operand arithmetic. - /// if is_imm true then it uses the imm field of rs2_or_imm, - /// otherwise it uses rs2 field. - /// - /// Used by e.g. add, sub - arithmetic_3op: struct { - is_imm: bool, - rd: Register, - rs1: Register, - rs2_or_imm: union { - rs2: Register, - imm: i13, - }, - }, - - /// Branch and link (always unconditional). - /// Used by e.g. call - branch_link: struct { - inst: Index, - // link is always %o7 - }, - - /// Branch with prediction, checking the integer status code - /// Used by e.g. bpcc - branch_predict_int: struct { - annul: bool = false, - pt: bool = true, - ccr: Instruction.CCR, - cond: Instruction.ICondition, - inst: Index, - }, - - /// Branch with prediction, comparing a register's content with zero - /// Used by e.g. bpr - branch_predict_reg: struct { - annul: bool = false, - pt: bool = true, - cond: Instruction.RCondition, - rs1: Register, - inst: Index, - }, - - /// ASI-tagged memory operations. - /// Used by e.g. ldxa, stxa - mem_asi: struct { - rd: Register, - rs1: Register, - rs2: Register = .g0, - asi: ASI, - }, - - /// Membar mask, controls the barrier behavior - /// Used by e.g. membar - membar_mask: struct { - mmask: Instruction.MemOrderingConstraint = .{}, - cmask: Instruction.MemCompletionConstraint = .{}, - }, - - /// Conditional move, checking the integer status code - /// if is_imm true then it uses the imm field of rs2_or_imm, - /// otherwise it uses rs2 field. - /// - /// Used by e.g. movcc - conditional_move_int: struct { - is_imm: bool, - ccr: Instruction.CCR, - cond: Instruction.Condition, - rd: Register, - rs2_or_imm: union { - rs2: Register, - imm: i11, - }, - }, - - /// Conditional move, comparing a register's content with zero - /// if is_imm true then it uses the imm field of rs2_or_imm, - /// otherwise it uses rs2 field. - /// - /// Used by e.g. movr - conditional_move_reg: struct { - is_imm: bool, - cond: Instruction.RCondition, - rd: Register, - rs1: Register, - rs2_or_imm: union { - rs2: Register, - imm: i10, - }, - }, - - /// No additional data - /// - /// Used by e.g. flushw - nop: void, - - /// SETHI operands. - /// - /// Used by sethi - sethi: struct { - rd: Register, - imm: u22, - }, - - /// Shift operands. - /// if is_imm true then it uses the imm field of rs2_or_imm, - /// otherwise it uses rs2 field. - /// - /// Used by e.g. sllx - shift: struct { - is_imm: bool, - rd: Register, - rs1: Register, - rs2_or_imm: union { - rs2: Register, - imm: u6, - }, - }, - - /// Trap. - /// if is_imm true then it uses the imm field of rs2_or_imm, - /// otherwise it uses rs2 field. - /// - /// Used by e.g. tcc - trap: struct { - is_imm: bool = true, - cond: Instruction.ICondition, - ccr: Instruction.CCR = .icc, - rs1: Register = .g0, - rs2_or_imm: union { - rs2: Register, - imm: u7, - }, - }, - }; - - // Make sure we don't accidentally make instructions bigger than expected. - // Note that in safety builds, Zig is allowed to insert a secret field for safety checks. - comptime { - if (!std.debug.runtime_safety) { - assert(@sizeOf(Data) == 8); - } - } -}; - -pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { - mir.instructions.deinit(gpa); - gpa.free(mir.extra); - mir.* = undefined; -} - -pub fn emit( - mir: Mir, - lf: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - atom_index: u32, - w: *std.Io.Writer, - debug_output: link.File.DebugInfoOutput, -) (codegen.CodeGenError || std.Io.Writer.Error)!void { - _ = atom_index; - const zcu = pt.zcu; - const func = zcu.funcInfo(func_index); - const nav = func.owner_nav; - const mod = zcu.navFileScope(nav).mod.?; - var e: Emit = .{ - .mir = mir, - .bin_file = lf, - .debug_output = debug_output, - .target = &mod.resolved_target.result, - .src_loc = src_loc, - .w = w, - .prev_di_pc = 0, - .prev_di_line = func.lbrace_line, - .prev_di_column = func.lbrace_column, - }; - defer e.deinit(); - e.emitMir() catch |err| switch (err) { - error.EmitFail => return zcu.codegenFailMsg(nav, e.err_msg.?), - else => |err1| return err1, - }; -} - -/// Returns the requested data, as well as the new index which is at the start of the -/// trailers for the object. -pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } { - const fields = std.meta.fields(T); - var i: usize = index; - var result: T = undefined; - inline for (fields) |field| { - @field(result, field.name) = switch (field.type) { - u32 => mir.extra[i], - i32 => @as(i32, @bitCast(mir.extra[i])), - else => @compileError("bad field type"), - }; - i += 1; - } - return .{ - .data = result, - .end = i, - }; -} diff --git a/src/arch/sparc64/abi.zig b/src/arch/sparc64/abi.zig deleted file mode 100644 index a26ab9a20b7db94620f33797b195cb9817d4467f..0000000000000000000000000000000000000000 --- a/src/arch/sparc64/abi.zig +++ /dev/null @@ -1,59 +0,0 @@ -const std = @import("std"); -const bits = @import("bits.zig"); -const Register = bits.Register; -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; - -// SPARCv9 SysV ABI stack constants. -// See: Registers and the Stack Frame, page 3P-8, SCD 2.4.1. - -// The ABI specifies that %sp points to top of stack - stack bias, -// and %fp points to top of previous frame - stack bias. -pub const stack_bias = 2047; - -// The first 128 bytes of the stack is reserved for register saving purposes. -// The ABI also requires to reserve space in the stack for the first six -// outgoing arguments, even though they are usually passed in registers. -// TODO Don't allocate the argument space in leaf functions -// TODO Save an RO copy of outgoing arguments in reserved area when building in Debug -// TODO Should we also save it in ReleaseSafe? Solaris and OpenBSD binaries seem to ship -// with argument copying enabled and it doesn't seem to give them big slowdowns so -// I guess it would be okay to do in ReleaseSafe? -pub const stack_reserved_area = 128 + 48; - -// There are no callee-preserved registers since the windowing -// mechanism already takes care of them. -// We still need to preserve %o0-%o5, %g1, %g4, and %g5 before calling -// something, though, as those are shared with the callee and might be -// thrashed by it. -pub const caller_preserved_regs = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5, .g1, .g4, .g5 }; - -// Try to allocate i, l, o, then g sets of registers, in order of priority. -const allocatable_regs = [_]Register{ - // zig fmt: off - .@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5", - .l0, .l1, .l2, .l3, .l4, .l5, .l6, .l7, - .o0, .o1, .o2, .o3, .o4, .o5, - .g1, .g4, .g5, - // zig fmt: on -}; - -pub const c_abi_int_param_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5 }; -pub const c_abi_int_param_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5" }; - -pub const c_abi_int_return_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3 }; -pub const c_abi_int_return_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3" }; - -pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs); - -// Register classes -const RegisterBitSet = RegisterManager.RegisterBitSet; -pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = 0, - .end = allocatable_regs.len, - }, true); - break :blk set; - }; -}; diff --git a/src/arch/sparc64/bits.zig b/src/arch/sparc64/bits.zig deleted file mode 100644 index d27371bf2505f2c7a4abde9255834f3063e896c2..0000000000000000000000000000000000000000 --- a/src/arch/sparc64/bits.zig +++ /dev/null @@ -1,1621 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; - -/// General purpose registers in the SPARCv9 instruction set -pub const Register = enum(u6) { - // zig fmt: off - g0, g1, g2, g3, g4, g5, g6, g7, - o0, o1, o2, o3, o4, o5, o6, o7, - l0, l1, l2, l3, l4, l5, l6, l7, - @"i0", @"i1", @"i2", @"i3", @"i4", @"i5", @"i6", @"i7", - - sp = 46, // stack pointer (o6) - fp = 62, // frame pointer (i6) - // zig fmt: on - - pub fn id(reg: Register) u5 { - return @truncate(@intFromEnum(reg)); - } - - pub fn enc(reg: Register) u5 { - // For integer registers, enc() == id(). - return reg.id(); - } - - pub fn dwarfNum(reg: Register) u5 { - return reg.id(); - } -}; - -test "Register.id" { - // SP - try testing.expectEqual(@as(u5, 14), Register.o6.id()); - try testing.expectEqual(Register.o6.id(), Register.sp.id()); - - // FP - try testing.expectEqual(@as(u5, 30), Register.i6.id()); - try testing.expectEqual(Register.i6.id(), Register.fp.id()); - - // x0 - try testing.expectEqual(@as(u5, 0), Register.g0.id()); - try testing.expectEqual(@as(u5, 8), Register.o0.id()); - try testing.expectEqual(@as(u5, 16), Register.l0.id()); - try testing.expectEqual(@as(u5, 24), Register.i0.id()); -} - -test "Register.enc" { - // x0 - try testing.expectEqual(@as(u5, 0), Register.g0.enc()); - try testing.expectEqual(@as(u5, 8), Register.o0.enc()); - try testing.expectEqual(@as(u5, 16), Register.l0.enc()); - try testing.expectEqual(@as(u5, 24), Register.i0.enc()); - - // For integer registers, enc() == id(). - try testing.expectEqual(Register.g0.enc(), Register.g0.id()); - try testing.expectEqual(Register.o0.enc(), Register.o0.id()); - try testing.expectEqual(Register.l0.enc(), Register.l0.id()); - try testing.expectEqual(Register.i0.enc(), Register.i0.id()); -} - -/// Scalar floating point registers in the SPARCv9 instruction set -pub const FloatingPointRegister = enum(u7) { - // SPARCv9 has 64 f32 registers, 32 f64 registers, and 16 f128 registers, - // which are aliased in this way: - // - // | %d0 | %d2 | - // %q0 | %f0 | %f1 | %f2 | %f3 | - // | %d4 | %d6 | - // %q4 | %f4 | %f5 | %f6 | %f7 | - // ... - // | %d60 | %d62 | - // %q60 | %f60 | %f61 | %f62 | %f63 | - // - // Though, since the instructions uses five-bit addressing, only %f0-%f31 - // is usable with f32 instructions. - - // zig fmt: off - - // 32-bit registers - @"f0", @"f1", @"f2", @"f3", @"f4", @"f5", @"f6", @"f7", - @"f8", @"f9", @"f10", @"f11", @"f12", @"f13", @"f14", @"f15", - @"f16", @"f17", @"f18", @"f19", @"f20", @"f21", @"f22", @"f23", - @"f24", @"f25", @"f26", @"f27", @"f28", @"f29", @"f30", @"f31", - - // 64-bit registers - d0, d2, d4, d6, d8, d10, d12, d14, - d16, d18, d20, d22, d24, d26, d28, d30, - d32, d34, d36, d38, d40, d42, d44, d46, - d48, d50, d52, d54, d56, d58, d60, d62, - - // 128-bit registers - q0, q4, q8, q12, q16, q20, q24, q28, - q32, q36, q40, q44, q48, q52, q56, q60, - // zig fmt: on - - pub fn id(self: FloatingPointRegister) u6 { - return switch (self.size()) { - 32 => @as(u6, @truncate(@intFromEnum(self))), - 64 => @as(u6, @truncate((@intFromEnum(self) - 32) * 2)), - 128 => @as(u6, @truncate((@intFromEnum(self) - 64) * 4)), - else => unreachable, - }; - } - - pub fn enc(self: FloatingPointRegister) u5 { - // Floating point registers use an encoding scheme to map from the 6-bit - // ID to 5-bit encoded value. - // (See section 5.1.4.1 of SPARCv9 ISA specification) - - const reg_id = self.id(); - return @as(u5, @truncate(reg_id | (reg_id >> 5))); - } - - /// Returns the bit-width of the register. - pub fn size(self: FloatingPointRegister) u8 { - return switch (@intFromEnum(self)) { - 0...31 => 32, - 32...63 => 64, - 64...79 => 128, - else => unreachable, - }; - } -}; - -test "FloatingPointRegister.id" { - // Low region - try testing.expectEqual(@as(u6, 0), FloatingPointRegister.q0.id()); - try testing.expectEqual(FloatingPointRegister.q0.id(), FloatingPointRegister.d0.id()); - try testing.expectEqual(FloatingPointRegister.d0.id(), FloatingPointRegister.f0.id()); - - try testing.expectEqual(@as(u6, 28), FloatingPointRegister.q28.id()); - try testing.expectEqual(FloatingPointRegister.q28.id(), FloatingPointRegister.d28.id()); - try testing.expectEqual(FloatingPointRegister.d28.id(), FloatingPointRegister.f28.id()); - - // High region - try testing.expectEqual(@as(u6, 32), FloatingPointRegister.q32.id()); - try testing.expectEqual(FloatingPointRegister.q32.id(), FloatingPointRegister.d32.id()); - - try testing.expectEqual(@as(u6, 60), FloatingPointRegister.q60.id()); - try testing.expectEqual(FloatingPointRegister.q60.id(), FloatingPointRegister.d60.id()); -} - -test "FloatingPointRegister.enc" { - // f registers - try testing.expectEqual(@as(u5, 0), FloatingPointRegister.f0.enc()); - try testing.expectEqual(@as(u5, 1), FloatingPointRegister.f1.enc()); - try testing.expectEqual(@as(u5, 31), FloatingPointRegister.f31.enc()); - - // d registers - try testing.expectEqual(@as(u5, 0), FloatingPointRegister.d0.enc()); - try testing.expectEqual(@as(u5, 1), FloatingPointRegister.d32.enc()); - try testing.expectEqual(@as(u5, 31), FloatingPointRegister.d62.enc()); - - // q registers - try testing.expectEqual(@as(u5, 0), FloatingPointRegister.q0.enc()); - try testing.expectEqual(@as(u5, 1), FloatingPointRegister.q32.enc()); - try testing.expectEqual(@as(u5, 29), FloatingPointRegister.q60.enc()); -} - -/// Represents an instruction in the SPARCv9 instruction set -pub const Instruction = union(enum) { - // Some of the instruction formats have several minor formats, here I - // name them with letters since there's no official naming scheme. - // TODO: need to rename the minor formats to a more descriptive name. - - // I am using regular structs instead of packed ones to avoid - // endianness-dependent behavior when constructing the actual - // assembly instructions. - // See also: https://github.com/ziglang/zig/issues/10113 - // TODO: change it back to packed structs once the issue is resolved. - - // Format 1 (op = 1): CALL - format_1: struct { - op: u2 = 0b01, - disp30: u30, - }, - - // Format 2 (op = 0): SETHI & Branches (Bicc, BPcc, BPr, FBfcc, FBPfcc) - format_2a: struct { - op: u2 = 0b00, - rd: u5, - op2: u3, - imm22: u22, - }, - format_2b: struct { - op: u2 = 0b00, - a: u1, - cond: u4, - op2: u3, - disp22: u22, - }, - format_2c: struct { - op: u2 = 0b00, - a: u1, - cond: u4, - op2: u3, - cc1: u1, - cc0: u1, - p: u1, - disp19: u19, - }, - format_2d: struct { - op: u2 = 0b00, - a: u1, - fixed: u1 = 0b0, - rcond: u3, - op2: u3, - d16hi: u2, - p: u1, - rs1: u5, - d16lo: u14, - }, - - // Format 3 (op = 2 or 3): Arithmetic, Logical, MOVr, MEMBAR, Load, and Store - format_3a: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b0, - reserved: u8 = 0b00000000, - rs2: u5, - }, - format_3b: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b1, - simm13: u13, - }, - format_3c: struct { - op: u2, - reserved1: u5 = 0b00000, - op3: u6, - rs1: u5, - i: u1 = 0b0, - reserved2: u8 = 0b00000000, - rs2: u5, - }, - format_3d: struct { - op: u2, - reserved: u5 = 0b00000, - op3: u6, - rs1: u5, - i: u1 = 0b1, - simm13: u13, - }, - format_3e: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b0, - rcond: u3, - reserved: u5 = 0b00000, - rs2: u5, - }, - format_3f: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b1, - rcond: u3, - simm10: u10, - }, - format_3g: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - // See Errata 58 of SPARCv9 specification - // https://sparc.org/errata-for-v9/#58 - i: u1 = 0b1, - reserved: u8 = 0b00000000, - rs2: u5, - }, - format_3h: struct { - op: u2 = 0b10, - fixed1: u5 = 0b00000, - op3: u6 = 0b101000, - fixed2: u5 = 0b01111, - i: u1 = 0b1, - reserved: u6 = 0b000000, - cmask: u3, - mmask: u4, - }, - format_3i: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b0, - imm_asi: u8, - rs2: u5, - }, - format_3j: struct { - op: u2, - impl_dep1: u5, - op3: u6, - impl_dep2: u19, - }, - format_3k: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b0, - x: u1, - reserved: u7 = 0b0000000, - rs2: u5, - }, - format_3l: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b1, - x: u1 = 0b0, - reserved: u7 = 0b0000000, - shcnt32: u5, - }, - format_3m: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b1, - x: u1 = 0b1, - reserved: u6 = 0b000000, - shcnt64: u6, - }, - format_3n: struct { - op: u2, - rd: u5, - op3: u6, - reserved: u5 = 0b00000, - opf: u9, - rs2: u5, - }, - format_3o: struct { - op: u2, - fixed: u3 = 0b000, - cc1: u1, - cc0: u1, - op3: u6, - rs1: u5, - opf: u9, - rs2: u5, - }, - format_3p: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - opf: u9, - rs2: u5, - }, - format_3q: struct { - op: u2, - rd: u5, - op3: u6, - rs1: u5, - reserved: u14 = 0b00000000000000, - }, - format_3r: struct { - op: u2, - fcn: u5, - op3: u6, - reserved: u19 = 0b0000000000000000000, - }, - format_3s: struct { - op: u2, - rd: u5, - op3: u6, - reserved: u19 = 0b0000000000000000000, - }, - - //Format 4 (op = 2): MOVcc, FMOVr, FMOVcc, and Tcc - format_4a: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b0, - cc1: u1, - cc0: u1, - reserved: u6 = 0b000000, - rs2: u5, - }, - format_4b: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b1, - cc1: u1, - cc0: u1, - simm11: u11, - }, - format_4c: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - cc2: u1, - cond: u4, - i: u1 = 0b0, - cc1: u1, - cc0: u1, - reserved: u6 = 0b000000, - rs2: u5, - }, - format_4d: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - cc2: u1, - cond: u4, - i: u1 = 0b1, - cc1: u1, - cc0: u1, - simm11: u11, - }, - format_4e: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - rs1: u5, - i: u1 = 0b1, - cc1: u1, - cc0: u1, - reserved: u4 = 0b0000, - sw_trap: u7, - }, - format_4f: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - rs1: u5, - fixed: u1 = 0b0, - rcond: u3, - opf_low: u5, - rs2: u5, - }, - format_4g: struct { - op: u2 = 0b10, - rd: u5, - op3: u6, - fixed: u1 = 0b0, - cond: u4, - opf_cc: u3, - opf_low: u6, - rs2: u5, - }, - - pub const CCR = enum(u3) { - fcc0, - fcc1, - fcc2, - fcc3, - icc, - reserved1, - xcc, - reserved2, - }; - - pub const RCondition = enum(u3) { - reserved1, - eq_zero, - le_zero, - lt_zero, - reserved2, - ne_zero, - gt_zero, - ge_zero, - - /// Returns the condition which is true iff the given condition is - /// false (if such a condition exists). - pub fn negate(cond: RCondition) RCondition { - return switch (cond) { - .eq_zero => .ne_zero, - .ne_zero => .eq_zero, - .lt_zero => .ge_zero, - .ge_zero => .lt_zero, - .le_zero => .gt_zero, - .gt_zero => .le_zero, - .reserved1 => unreachable, - .reserved2 => unreachable, - }; - } - }; - - pub const ASI = enum(u8) { - asi_nucleus = 0x04, - asi_nucleus_little = 0x0c, - asi_as_if_user_primary = 0x10, - asi_as_if_user_secondary = 0x11, - asi_as_if_user_primary_little = 0x18, - asi_as_if_user_secondary_little = 0x19, - asi_primary = 0x80, - asi_secondary = 0x81, - asi_primary_nofault = 0x82, - asi_secondary_nofault = 0x83, - asi_primary_little = 0x88, - asi_secondary_little = 0x89, - asi_primary_nofault_little = 0x8a, - asi_secondary_nofault_little = 0x8b, - }; - - pub const ShiftWidth = enum(u1) { - shift32, - shift64, - }; - - pub const MemOrderingConstraint = packed struct { - store_store: bool = false, - load_store: bool = false, - store_load: bool = false, - load_load: bool = false, - }; - - pub const MemCompletionConstraint = packed struct { - sync: bool = false, - mem_issue: bool = false, - lookaside: bool = false, - }; - - // In SPARCv9, FP and integer comparison operations - // are encoded differently. - - pub const FCondition = enum(u4) { - /// Branch Never - nv, - /// Branch on Not Equal - ne, - /// Branch on Less or Greater - lg, - /// Branch on Unordered or Less - ul, - /// Branch on Less - lt, - /// Branch on Unordered or Greater - ug, - /// Branch on Greater - gt, - /// Branch on Unordered - un, - /// Branch Always - al, - /// Branch on Equal - eq, - /// Branch on Unordered or Equal - ue, - /// Branch on Greater or Equal - ge, - /// Branch on Unordered or Greater or Equal - uge, - /// Branch on Less or Equal - le, - /// Branch on Unordered or Less or Equal - ule, - /// Branch on Ordered - ord, - - /// Converts a std.math.CompareOperator into a condition flag, - /// i.e. returns the condition that is true iff the result of the - /// comparison is true. - pub fn fromCompareOperator(op: std.math.CompareOperator) FCondition { - return switch (op) { - .gte => .ge, - .gt => .gt, - .neq => .ne, - .lt => .lt, - .lte => .le, - .eq => .eq, - }; - } - - /// Returns the condition which is true iff the given condition is - /// false (if such a condition exists). - pub fn negate(cond: FCondition) FCondition { - return switch (cond) { - .eq => .ne, - .ne => .eq, - .ge => .ul, - .ul => .ge, - .le => .ug, - .ug => .le, - .lt => .uge, - .uge => .lt, - .gt => .ule, - .ule => .gt, - .ue => .lg, - .lg => .ue, - .ord => .un, - .un => .ord, - .al => unreachable, - .nv => unreachable, - }; - } - }; - - pub const ICondition = enum(u4) { - /// Branch Never - nv, - /// Branch on Equal - eq, - /// Branch on Less or Equal - le, - /// Branch on Less - lt, - /// Branch on Less or Equal Unsigned - leu, - /// Branch on Carry Set (Less than, Unsigned) - cs, - /// Branch on Negative - neg, - /// Branch on Overflow Set - vs, - /// Branch Always - al, - /// Branch on Not Equal - ne, - /// Branch on Greater - gt, - /// Branch on Greater or Equal - ge, - /// Branch on Greater Unsigned - gu, - /// Branch on Carry Clear (Greater Than or Equal, Unsigned) - cc, - /// Branch on Positive - pos, - /// Branch on Overflow Clear - vc, - - /// Converts a std.math.CompareOperator into a condition flag, - /// i.e. returns the condition that is true iff the result of the - /// comparison is true. Assumes signed comparison. - pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) ICondition { - return switch (op) { - .gte => .ge, - .gt => .gt, - .neq => .ne, - .lt => .lt, - .lte => .le, - .eq => .eq, - }; - } - - /// Converts a std.math.CompareOperator into a condition flag, - /// i.e. returns the condition that is true iff the result of the - /// comparison is true. Assumes unsigned comparison. - pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) ICondition { - return switch (op) { - .gte => .cc, - .gt => .gu, - .neq => .ne, - .lt => .cs, - .lte => .leu, - .eq => .eq, - }; - } - - /// Returns the condition which is true iff the given condition is - /// false (if such a condition exists). - pub fn negate(cond: ICondition) ICondition { - return switch (cond) { - .eq => .ne, - .ne => .eq, - .cs => .cc, - .cc => .cs, - .neg => .pos, - .pos => .neg, - .vs => .vc, - .vc => .vs, - .gu => .leu, - .leu => .gu, - .ge => .lt, - .lt => .ge, - .gt => .le, - .le => .gt, - .al => unreachable, - .nv => unreachable, - }; - } - }; - - pub const ConditionTag = enum { fcond, icond }; - pub const Condition = union(ConditionTag) { - fcond: FCondition, - icond: ICondition, - - /// Encodes the condition into the instruction bit pattern. - pub fn enc(cond: Condition) u4 { - return switch (cond) { - .icond => |c| @intFromEnum(c), - .fcond => |c| @intFromEnum(c), - }; - } - - /// Returns the condition which is true iff the given condition is - /// false (if such a condition exists). - pub fn negate(cond: Condition) Condition { - return switch (cond) { - .icond => |c| .{ .icond = c.negate() }, - .fcond => |c| .{ .fcond = c.negate() }, - }; - } - }; - - pub fn toU32(self: Instruction) u32 { - // TODO: Remove this once packed structs work. - return switch (self) { - .format_1 => |v| (@as(u32, v.op) << 30) | @as(u32, v.disp30), - .format_2a => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op2) << 22) | @as(u32, v.imm22), - .format_2b => |v| (@as(u32, v.op) << 30) | (@as(u32, v.a) << 29) | (@as(u32, v.cond) << 25) | (@as(u32, v.op2) << 22) | @as(u32, v.disp22), - .format_2c => |v| (@as(u32, v.op) << 30) | (@as(u32, v.a) << 29) | (@as(u32, v.cond) << 25) | (@as(u32, v.op2) << 22) | (@as(u32, v.cc1) << 21) | (@as(u32, v.cc0) << 20) | (@as(u32, v.p) << 19) | @as(u32, v.disp19), - .format_2d => |v| (@as(u32, v.op) << 30) | (@as(u32, v.a) << 29) | (@as(u32, v.fixed) << 28) | (@as(u32, v.rcond) << 25) | (@as(u32, v.op2) << 22) | (@as(u32, v.d16hi) << 20) | (@as(u32, v.p) << 19) | (@as(u32, v.rs1) << 14) | @as(u32, v.d16lo), - .format_3a => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), - .format_3b => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | @as(u32, v.simm13), - .format_3c => |v| (@as(u32, v.op) << 30) | (@as(u32, v.reserved1) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved2) << 5) | @as(u32, v.rs2), - .format_3d => |v| (@as(u32, v.op) << 30) | (@as(u32, v.reserved) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | @as(u32, v.simm13), - .format_3e => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.rcond) << 10) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), - .format_3f => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.rcond) << 10) | @as(u32, v.simm10), - .format_3g => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), - .format_3h => |v| (@as(u32, v.op) << 30) | (@as(u32, v.fixed1) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.fixed2) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved) << 7) | (@as(u32, v.cmask) << 4) | @as(u32, v.mmask), - .format_3i => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.imm_asi) << 5) | @as(u32, v.rs2), - .format_3j => |v| (@as(u32, v.op) << 30) | (@as(u32, v.impl_dep1) << 25) | (@as(u32, v.op3) << 19) | @as(u32, v.impl_dep2), - .format_3k => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.x) << 12) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), - .format_3l => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.x) << 12) | (@as(u32, v.reserved) << 5) | @as(u32, v.shcnt32), - .format_3m => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.x) << 12) | (@as(u32, v.reserved) << 6) | @as(u32, v.shcnt64), - .format_3n => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.reserved) << 14) | (@as(u32, v.opf) << 5) | @as(u32, v.rs2), - .format_3o => |v| (@as(u32, v.op) << 30) | (@as(u32, v.fixed) << 27) | (@as(u32, v.cc1) << 26) | (@as(u32, v.cc0) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.opf) << 5) | @as(u32, v.rs2), - .format_3p => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.opf) << 5) | @as(u32, v.rs2), - .format_3q => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | @as(u32, v.reserved), - .format_3r => |v| (@as(u32, v.op) << 30) | (@as(u32, v.fcn) << 25) | (@as(u32, v.op3) << 19) | @as(u32, v.reserved), - .format_3s => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | @as(u32, v.reserved), - .format_4a => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), - .format_4b => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | @as(u32, v.simm11), - .format_4c => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.cc2) << 18) | (@as(u32, v.cond) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), - .format_4d => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.cc2) << 18) | (@as(u32, v.cond) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | @as(u32, v.simm11), - .format_4e => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | (@as(u32, v.reserved) << 7) | @as(u32, v.sw_trap), - .format_4f => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.fixed) << 13) | (@as(u32, v.rcond) << 10) | (@as(u32, v.opf_low) << 5) | @as(u32, v.rs2), - .format_4g => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.fixed) << 18) | (@as(u32, v.cond) << 14) | (@as(u32, v.opf_cc) << 11) | (@as(u32, v.opf_low) << 5) | @as(u32, v.rs2), - }; - } - - // SPARCv9 Instruction formats. - // See section 6.2 of the SPARCv9 ISA manual. - - fn format1(disp: i32) Instruction { - const udisp = @as(u32, @bitCast(disp)); - - // In SPARC, branch target needs to be aligned to 4 bytes. - assert(udisp % 4 == 0); - - // Discard the last two bits since those are implicitly zero. - const udisp_truncated = @as(u30, @truncate(udisp >> 2)); - return Instruction{ - .format_1 = .{ - .disp30 = udisp_truncated, - }, - }; - } - - fn format2a(op2: u3, imm: u22, rd: Register) Instruction { - return Instruction{ - .format_2a = .{ - .rd = rd.enc(), - .op2 = op2, - .imm22 = imm, - }, - }; - } - - fn format2b(op2: u3, cond: Condition, annul: bool, disp: i24) Instruction { - const udisp = @as(u24, @bitCast(disp)); - - // In SPARC, branch target needs to be aligned to 4 bytes. - assert(udisp % 4 == 0); - - // Discard the last two bits since those are implicitly zero. - const udisp_truncated = @as(u22, @truncate(udisp >> 2)); - return Instruction{ - .format_2b = .{ - .a = @intFromBool(annul), - .cond = cond.enc(), - .op2 = op2, - .disp22 = udisp_truncated, - }, - }; - } - - fn format2c(op2: u3, cond: Condition, annul: bool, pt: bool, ccr: CCR, disp: i21) Instruction { - const udisp = @as(u21, @bitCast(disp)); - - // In SPARC, branch target needs to be aligned to 4 bytes. - assert(udisp % 4 == 0); - - // Discard the last two bits since those are implicitly zero. - const udisp_truncated = @as(u19, @truncate(udisp >> 2)); - - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_2c = .{ - .a = @intFromBool(annul), - .cond = cond.enc(), - .op2 = op2, - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .p = @intFromBool(pt), - .disp19 = udisp_truncated, - }, - }; - } - - fn format2d(op2: u3, rcond: RCondition, annul: bool, pt: bool, rs1: Register, disp: i18) Instruction { - const udisp = @as(u18, @bitCast(disp)); - - // In SPARC, branch target needs to be aligned to 4 bytes. - assert(udisp % 4 == 0); - - // Discard the last two bits since those are implicitly zero, - // and split it into low and high parts. - const udisp_truncated = @as(u16, @truncate(udisp >> 2)); - const udisp_hi = @as(u2, @truncate((udisp_truncated & 0b1100_0000_0000_0000) >> 14)); - const udisp_lo = @as(u14, @truncate(udisp_truncated & 0b0011_1111_1111_1111)); - return Instruction{ - .format_2d = .{ - .a = @intFromBool(annul), - .rcond = @intFromEnum(rcond), - .op2 = op2, - .p = @intFromBool(pt), - .rs1 = rs1.enc(), - .d16hi = udisp_hi, - .d16lo = udisp_lo, - }, - }; - } - - fn format3a(op: u2, op3: u6, rs1: Register, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_3a = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .rs2 = rs2.enc(), - }, - }; - } - fn format3b(op: u2, op3: u6, rs1: Register, imm: i13, rd: Register) Instruction { - return Instruction{ - .format_3b = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .simm13 = @as(u13, @bitCast(imm)), - }, - }; - } - fn format3c(op: u2, op3: u6, rs1: Register, rs2: Register) Instruction { - return Instruction{ - .format_3c = .{ - .op = op, - .op3 = op3, - .rs1 = rs1.enc(), - .rs2 = rs2.enc(), - }, - }; - } - fn format3d(op: u2, op3: u6, rs1: Register, imm: i13) Instruction { - return Instruction{ - .format_3d = .{ - .op = op, - .op3 = op3, - .rs1 = rs1.enc(), - .simm13 = @as(u13, @bitCast(imm)), - }, - }; - } - fn format3e(op: u2, op3: u6, rcond: RCondition, rs1: Register, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_3e = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .rcond = @intFromEnum(rcond), - .rs2 = rs2.enc(), - }, - }; - } - fn format3f(op: u2, op3: u6, rcond: RCondition, rs1: Register, imm: i10, rd: Register) Instruction { - return Instruction{ - .format_3f = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .rcond = @intFromEnum(rcond), - .simm10 = @as(u10, @bitCast(imm)), - }, - }; - } - fn format3g(op: u2, op3: u6, rs1: Register, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_3g = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .rs2 = rs2.enc(), - }, - }; - } - fn format3h(cmask: MemCompletionConstraint, mmask: MemOrderingConstraint) Instruction { - return Instruction{ - .format_3h = .{ - .cmask = @as(u3, @bitCast(cmask)), - .mmask = @as(u4, @bitCast(mmask)), - }, - }; - } - fn format3i(op: u2, op3: u6, rs1: Register, rs2: Register, rd: Register, asi: ASI) Instruction { - return Instruction{ - .format_3i = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .imm_asi = @intFromEnum(asi), - .rs2 = rs2.enc(), - }, - }; - } - fn format3j(op: u2, op3: u6, impl_dep1: u5, impl_dep2: u19) Instruction { - return Instruction{ - .format_3j = .{ - .op = op, - .impl_dep1 = impl_dep1, - .op3 = op3, - .impl_dep2 = impl_dep2, - }, - }; - } - fn format3k(op: u2, op3: u6, sw: ShiftWidth, rs1: Register, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_3k = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .x = @intFromEnum(sw), - .rs2 = rs2.enc(), - }, - }; - } - fn format3l(op: u2, op3: u6, rs1: Register, shift_count: u5, rd: Register) Instruction { - return Instruction{ - .format_3l = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .shcnt32 = shift_count, - }, - }; - } - fn format3m(op: u2, op3: u6, rs1: Register, shift_count: u6, rd: Register) Instruction { - return Instruction{ - .format_3m = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .shcnt64 = shift_count, - }, - }; - } - fn format3n(op: u2, op3: u6, opf: u9, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_3n = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .opf = opf, - .rs2 = rs2.enc(), - }, - }; - } - fn format3o(op: u2, op3: u6, opf: u9, ccr: CCR, rs1: Register, rs2: Register) Instruction { - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_3o = .{ - .op = op, - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .op3 = op3, - .rs1 = rs1.enc(), - .opf = opf, - .rs2 = rs2.enc(), - }, - }; - } - fn format3p(op: u2, op3: u6, opf: u9, rs1: Register, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_3p = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .opf = opf, - .rs2 = rs2.enc(), - }, - }; - } - fn format3q(op: u2, op3: u6, rs1: Register, rd: Register) Instruction { - return Instruction{ - .format_3q = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - }, - }; - } - fn format3r(op: u2, op3: u6, fcn: u5) Instruction { - return Instruction{ - .format_3r = .{ - .op = op, - .fcn = fcn, - .op3 = op3, - }, - }; - } - fn format3s(op: u2, op3: u6, rd: Register) Instruction { - return Instruction{ - .format_3s = .{ - .op = op, - .rd = rd.enc(), - .op3 = op3, - }, - }; - } - - fn format4a(op3: u6, ccr: CCR, rs1: Register, rs2: Register, rd: Register) Instruction { - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_4a = .{ - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .rs2 = rs2.enc(), - }, - }; - } - - fn format4b(op3: u6, ccr: CCR, rs1: Register, imm: i11, rd: Register) Instruction { - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_4b = .{ - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .simm11 = @as(u11, @bitCast(imm)), - }, - }; - } - - fn format4c(op3: u6, cond: Condition, ccr: CCR, rs2: Register, rd: Register) Instruction { - const ccr_cc2 = @as(u1, @truncate(@intFromEnum(ccr) >> 2)); - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_4c = .{ - .rd = rd.enc(), - .op3 = op3, - .cc2 = ccr_cc2, - .cond = cond.enc(), - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .rs2 = rs2.enc(), - }, - }; - } - - fn format4d(op3: u6, cond: Condition, ccr: CCR, imm: i11, rd: Register) Instruction { - const ccr_cc2 = @as(u1, @truncate(@intFromEnum(ccr) >> 2)); - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_4d = .{ - .rd = rd.enc(), - .op3 = op3, - .cc2 = ccr_cc2, - .cond = cond.enc(), - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .simm11 = @as(u11, @bitCast(imm)), - }, - }; - } - - fn format4e(op3: u6, ccr: CCR, rs1: Register, rd: Register, sw_trap: u7) Instruction { - const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); - const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); - return Instruction{ - .format_4e = .{ - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .cc1 = ccr_cc1, - .cc0 = ccr_cc0, - .sw_trap = sw_trap, - }, - }; - } - - fn format4f( - op3: u6, - opf_low: u5, - rcond: RCondition, - rs1: Register, - rs2: Register, - rd: Register, - ) Instruction { - return Instruction{ - .format_4f = .{ - .rd = rd.enc(), - .op3 = op3, - .rs1 = rs1.enc(), - .rcond = @intFromEnum(rcond), - .opf_low = opf_low, - .rs2 = rs2.enc(), - }, - }; - } - - fn format4g(op3: u6, opf_low: u6, opf_cc: u3, cond: Condition, rs2: Register, rd: Register) Instruction { - return Instruction{ - .format_4g = .{ - .rd = rd.enc(), - .op3 = op3, - .cond = cond.enc(), - .opf_cc = opf_cc, - .opf_low = opf_low, - .rs2 = rs2.enc(), - }, - }; - } - - // SPARCv9 Instruction definition. - // See appendix A of the SPARCv9 ISA manual. - - pub fn add(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_0000, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_0000, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn addcc(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b01_0000, rs1, rs2, rd), - i13 => format3b(0b10, 0b01_0000, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn bpcc(cond: ICondition, annul: bool, pt: bool, ccr: CCR, disp: i21) Instruction { - return format2c(0b001, .{ .icond = cond }, annul, pt, ccr, disp); - } - - pub fn bpr(cond: RCondition, annul: bool, pt: bool, rs1: Register, disp: i18) Instruction { - return format2d(0b011, cond, annul, pt, rs1, disp); - } - - pub fn jmpl(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b11_1000, rs1, rs2, rd), - i13 => format3b(0b10, 0b11_1000, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn ldub(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_0001, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_0001, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn lduh(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_0010, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_0010, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn lduw(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_0000, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_0000, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn ldx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_1011, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_1011, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn lduba(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_0001, rs1, rs2, rd, asi); - } - - pub fn lduha(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_0010, rs1, rs2, rd, asi); - } - - pub fn lduwa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_0000, rs1, rs2, rd, asi); - } - - pub fn ldxa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_1011, rs1, rs2, rd, asi); - } - - pub fn @"and"(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_0001, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_0001, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn @"or"(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_0010, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_0010, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn xor(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_0011, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_0011, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn xnor(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_0111, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_0111, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn membar(cmask: MemCompletionConstraint, mmask: MemOrderingConstraint) Instruction { - return format3h(cmask, mmask); - } - - pub fn movcc(comptime s2: type, cond: Condition, ccr: CCR, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format4c(0b10_1100, cond, ccr, rs2, rd), - i11 => format4d(0b10_1100, cond, ccr, rs2, rd), - else => unreachable, - }; - } - - pub fn movr(comptime s2: type, cond: RCondition, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3e(0b10, 0b10_1111, cond, rs1, rs2, rd), - i10 => format3f(0b10, 0b10_1111, cond, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn mulx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_1001, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_1001, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn sdivx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b10_1101, rs1, rs2, rd), - i13 => format3b(0b10, 0b10_1101, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn udivx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_1101, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_1101, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn nop() Instruction { - return sethi(0, .g0); - } - - pub fn @"return"(comptime s2: type, rs1: Register, rs2: s2) Instruction { - return switch (s2) { - Register => format3c(0b10, 0b11_1001, rs1, rs2), - i13 => format3d(0b10, 0b11_1001, rs1, rs2), - else => unreachable, - }; - } - - pub fn save(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b11_1100, rs1, rs2, rd), - i13 => format3b(0b10, 0b11_1100, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn restore(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b11_1101, rs1, rs2, rd), - i13 => format3b(0b10, 0b11_1101, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn sethi(imm: u22, rd: Register) Instruction { - return format2a(0b100, imm, rd); - } - - pub fn sll(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3k(0b10, 0b10_0101, .shift32, rs1, rs2, rd), - u5 => format3l(0b10, 0b10_0101, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn srl(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3k(0b10, 0b10_0110, .shift32, rs1, rs2, rd), - u5 => format3l(0b10, 0b10_0110, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn sra(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3k(0b10, 0b10_0111, .shift32, rs1, rs2, rd), - u5 => format3l(0b10, 0b10_0111, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn sllx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3k(0b10, 0b10_0101, .shift64, rs1, rs2, rd), - u6 => format3m(0b10, 0b10_0101, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn srlx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3k(0b10, 0b10_0110, .shift64, rs1, rs2, rd), - u6 => format3m(0b10, 0b10_0110, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn srax(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3k(0b10, 0b10_0111, .shift64, rs1, rs2, rd), - u6 => format3m(0b10, 0b10_0111, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn stb(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_0101, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_0101, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn sth(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_0110, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_0110, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn stw(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_0100, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_0100, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn stx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b11, 0b00_1110, rs1, rs2, rd), - i13 => format3b(0b11, 0b00_1110, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn stba(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_0101, rs1, rs2, rd, asi); - } - - pub fn stha(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_0110, rs1, rs2, rd, asi); - } - - pub fn stwa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_0100, rs1, rs2, rd, asi); - } - - pub fn stxa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { - return format3i(0b11, 0b01_1110, rs1, rs2, rd, asi); - } - - pub fn sub(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b00_0100, rs1, rs2, rd), - i13 => format3b(0b10, 0b00_0100, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn subcc(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { - return switch (s2) { - Register => format3a(0b10, 0b01_0100, rs1, rs2, rd), - i13 => format3b(0b10, 0b01_0100, rs1, rs2, rd), - else => unreachable, - }; - } - - pub fn trap(comptime s2: type, cond: ICondition, ccr: CCR, rs1: Register, rs2: s2) Instruction { - // Tcc instructions abuse the rd field to store the conditionals. - return switch (s2) { - Register => format4a(0b11_1010, ccr, rs1, rs2, @as(Register, @enumFromInt(@intFromEnum(cond)))), - u7 => format4e(0b11_1010, ccr, rs1, @as(Register, @enumFromInt(@intFromEnum(cond))), rs2), - else => unreachable, - }; - } -}; - -test "Serialize formats" { - const Testcase = struct { - inst: Instruction, - expected: u32, - }; - - // Note that the testcases might or might not be a valid instruction - // This is mostly just to check the behavior of the format packed structs - // since currently stage1 doesn't properly implement it in all cases - const testcases = [_]Testcase{ - .{ - .inst = Instruction.format1(4), - .expected = 0b01_000000000000000000000000000001, - }, - .{ - .inst = Instruction.format2a(4, 0, .g0), - .expected = 0b00_00000_100_0000000000000000000000, - }, - .{ - .inst = Instruction.format2b(6, .{ .icond = .lt }, true, -4), - .expected = 0b00_1_0011_110_1111111111111111111111, - }, - .{ - .inst = Instruction.format2c(3, .{ .icond = .nv }, false, true, .xcc, 8), - .expected = 0b00_0_0000_011_1_0_1_0000000000000000010, - }, - .{ - .inst = Instruction.format2d(7, .eq_zero, false, true, .o0, 20), - .expected = 0b00_0_0_001_111_00_1_01000_00000000000101, - }, - .{ - .inst = Instruction.format3a(3, 5, .g0, .o1, .l2), - .expected = 0b11_10010_000101_00000_0_00000000_01001, - }, - .{ - .inst = Instruction.format3b(3, 5, .g0, -1, .l2), - .expected = 0b11_10010_000101_00000_1_1111111111111, - }, - .{ - .inst = Instruction.format3c(3, 5, .g0, .o1), - .expected = 0b11_00000_000101_00000_0_00000000_01001, - }, - .{ - .inst = Instruction.format3d(3, 5, .g0, 0), - .expected = 0b11_00000_000101_00000_1_0000000000000, - }, - .{ - .inst = Instruction.format3e(3, 5, .ne_zero, .g0, .o1, .l2), - .expected = 0b11_10010_000101_00000_0_101_00000_01001, - }, - .{ - .inst = Instruction.format3f(3, 5, .ne_zero, .g0, -1, .l2), - .expected = 0b11_10010_000101_00000_1_101_1111111111, - }, - .{ - .inst = Instruction.format3g(3, 5, .g0, .o1, .l2), - .expected = 0b11_10010_000101_00000_1_00000000_01001, - }, - .{ - .inst = Instruction.format3h(.{}, .{}), - .expected = 0b10_00000_101000_01111_1_000000_000_0000, - }, - .{ - .inst = Instruction.format3i(3, 5, .g0, .o1, .l2, .asi_primary_little), - .expected = 0b11_10010_000101_00000_0_10001000_01001, - }, - .{ - .inst = Instruction.format3j(3, 5, 31, 0), - .expected = 0b11_11111_000101_0000000000000000000, - }, - .{ - .inst = Instruction.format3k(3, 5, .shift32, .g0, .o1, .l2), - .expected = 0b11_10010_000101_00000_0_0_0000000_01001, - }, - .{ - .inst = Instruction.format3l(3, 5, .g0, 31, .l2), - .expected = 0b11_10010_000101_00000_1_0_0000000_11111, - }, - .{ - .inst = Instruction.format3m(3, 5, .g0, 63, .l2), - .expected = 0b11_10010_000101_00000_1_1_000000_111111, - }, - .{ - .inst = Instruction.format3n(3, 5, 0, .o1, .l2), - .expected = 0b11_10010_000101_00000_000000000_01001, - }, - .{ - .inst = Instruction.format3o(3, 5, 0, .xcc, .o1, .l2), - .expected = 0b11_000_1_0_000101_01001_000000000_10010, - }, - .{ - .inst = Instruction.format3p(3, 5, 0, .g0, .o1, .l2), - .expected = 0b11_10010_000101_00000_000000000_01001, - }, - .{ - .inst = Instruction.format3q(3, 5, .g0, .o1), - .expected = 0b11_01001_000101_00000_00000000000000, - }, - .{ - .inst = Instruction.format3r(3, 5, 4), - .expected = 0b11_00100_000101_0000000000000000000, - }, - .{ - .inst = Instruction.format3s(3, 5, .g0), - .expected = 0b11_00000_000101_0000000000000000000, - }, - .{ - .inst = Instruction.format4a(8, .xcc, .g0, .o1, .l2), - .expected = 0b10_10010_001000_00000_0_1_0_000000_01001, - }, - .{ - .inst = Instruction.format4b(8, .xcc, .g0, -1, .l2), - .expected = 0b10_10010_001000_00000_1_1_0_11111111111, - }, - .{ - .inst = Instruction.format4c(8, .{ .icond = .nv }, .xcc, .g0, .o1), - .expected = 0b10_01001_001000_1_0000_0_1_0_000000_00000, - }, - .{ - .inst = Instruction.format4d(8, .{ .icond = .nv }, .xcc, 0, .l2), - .expected = 0b10_10010_001000_1_0000_1_1_0_00000000000, - }, - .{ - .inst = Instruction.format4e(8, .xcc, .g0, .o1, 0), - .expected = 0b10_01001_001000_00000_1_1_0_0000_0000000, - }, - .{ - .inst = Instruction.format4f(8, 4, .eq_zero, .g0, .o1, .l2), - .expected = 0b10_10010_001000_00000_0_001_00100_01001, - }, - .{ - .inst = Instruction.format4g(8, 4, 2, .{ .icond = .nv }, .o1, .l2), - .expected = 0b10_10010_001000_0_0000_010_000100_01001, - }, - }; - - for (testcases) |case| { - const actual = case.inst.toU32(); - testing.expectEqual(case.expected, actual) catch |err| { - std.debug.print("error: {}\n", .{err}); - std.debug.print("case: {}\n", .{case}); - return err; - }; - } -} diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig deleted file mode 100644 index d8d8933cc3ea780465a18f3bedf6ed0eefb821d1..0000000000000000000000000000000000000000 --- a/src/arch/wasm/CodeGen.zig +++ /dev/null @@ -1,7578 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const testing = std.testing; -const mem = std.mem; -const log = std.log.scoped(.codegen); - -const CodeGen = @This(); -const codegen = @import("../../codegen.zig"); -const Zcu = @import("../../Zcu.zig"); -const InternPool = @import("../../InternPool.zig"); -const Decl = Zcu.Decl; -const Type = @import("../../Type.zig"); -const Value = @import("../../Value.zig"); -const Compilation = @import("../../Compilation.zig"); -const link = @import("../../link.zig"); -const Air = @import("../../Air.zig"); -const Mir = @import("Mir.zig"); -const abi = @import("../../codegen/wasm/abi.zig"); -const Alignment = InternPool.Alignment; -const errUnionPayloadOffset = codegen.errUnionPayloadOffset; -const errUnionErrorOffset = codegen.errUnionErrorOffset; - -const target_util = @import("../../target.zig"); -const libcFloatPrefix = target_util.libcFloatPrefix; -const libcFloatSuffix = target_util.libcFloatSuffix; -const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev; -const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev; - -pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { - return comptime &.initMany(&.{ - .expand_intcast_safe, - .expand_int_from_float_safe, - .expand_int_from_float_optimized_safe, - .expand_add_safe, - .expand_sub_safe, - .expand_mul_safe, - }); -} - -/// Reference to the function declaration the code -/// section belongs to -owner_nav: InternPool.Nav.Index, -/// Current block depth. Used to calculate the relative difference between a break -/// and block -block_depth: u32 = 0, -air: Air, -liveness: Air.Liveness, -gpa: mem.Allocator, -func_index: InternPool.Index, -/// Contains a list of current branches. -/// When we return from a branch, the branch will be popped from this list, -/// which means branches can only contain references from within its own branch, -/// or a branch higher (lower index) in the tree. -branches: std.ArrayListUnmanaged(Branch) = .empty, -/// Table to save `WValue`'s generated by an `Air.Inst` -// values: ValueTable, -/// Mapping from Air.Inst.Index to block ids -blocks: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, struct { - label: u32, - value: WValue, -}) = .{}, -/// Maps `loop` instructions to their label. `br` to here repeats the loop. -loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty, -/// The index the next local generated will have -/// NOTE: arguments share the index with locals therefore the first variable -/// will have the index that comes after the last argument's index -local_index: u32, -/// The index of the current argument. -/// Used to track which argument is being referenced in `airArg`. -arg_index: u32 = 0, -/// List of simd128 immediates. Each value is stored as an array of bytes. -/// This list will only be populated for 128bit-simd values when the target features -/// are enabled also. -simd_immediates: std.ArrayListUnmanaged([16]u8) = .empty, -/// The Target we're emitting (used to call intInfo) -target: *const std.Target, -ptr_size: enum { wasm32, wasm64 }, -pt: Zcu.PerThread, -/// List of MIR Instructions -mir_instructions: std.MultiArrayList(Mir.Inst), -/// Contains extra data for MIR -mir_extra: std.ArrayListUnmanaged(u32), -/// List of all locals' types generated throughout this declaration -/// used to emit locals count at start of 'code' section. -mir_locals: std.ArrayListUnmanaged(std.wasm.Valtype), -/// Set of all UAVs referenced by this function. Key is the UAV value, value is the alignment. -/// `.none` means naturally aligned. An explicit alignment is never less than the natural alignment. -mir_uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment), -/// Set of all functions whose address this function has taken and which therefore might be called -/// via a `call_indirect` function. -mir_indirect_function_set: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, void), -/// Set of all function types used by this function. These must be interned by the linker. -mir_func_tys: std.AutoArrayHashMapUnmanaged(InternPool.Index, void), -/// The number of `error_name_table_ref` instructions emitted. -error_name_table_ref_count: u32, -/// When a function is executing, we store the the current stack pointer's value within this local. -/// This value is then used to restore the stack pointer to the original value at the return of the function. -initial_stack_value: WValue = .none, -/// The current stack pointer subtracted with the stack size. From this value, we will calculate -/// all offsets of the stack values. -bottom_stack_value: WValue = .none, -/// Arguments of this function declaration -/// This will be set after `resolveCallingConventionValues` -args: []WValue, -/// This will only be `.none` if the function returns void, or returns an immediate. -/// When it returns a pointer to the stack, the `.local` tag will be active and must be populated -/// before this function returns its execution to the caller. -return_value: WValue, -/// The size of the stack this function occupies. In the function prologue -/// we will move the stack pointer by this number, forward aligned with the `stack_alignment`. -stack_size: u32 = 0, -/// The stack alignment, which is 16 bytes by default. This is specified by the -/// tool-conventions: https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md -/// and also what the llvm backend will emit. -/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default. -stack_alignment: Alignment = .@"16", - -// For each individual Wasm valtype we store a seperate free list which -// allows us to re-use locals that are no longer used. e.g. a temporary local. -/// A list of indexes which represents a local of valtype `i32`. -/// It is illegal to store a non-i32 valtype in this list. -free_locals_i32: std.ArrayListUnmanaged(u32) = .empty, -/// A list of indexes which represents a local of valtype `i64`. -/// It is illegal to store a non-i64 valtype in this list. -free_locals_i64: std.ArrayListUnmanaged(u32) = .empty, -/// A list of indexes which represents a local of valtype `f32`. -/// It is illegal to store a non-f32 valtype in this list. -free_locals_f32: std.ArrayListUnmanaged(u32) = .empty, -/// A list of indexes which represents a local of valtype `f64`. -/// It is illegal to store a non-f64 valtype in this list. -free_locals_f64: std.ArrayListUnmanaged(u32) = .empty, -/// A list of indexes which represents a local of valtype `v127`. -/// It is illegal to store a non-v128 valtype in this list. -free_locals_v128: std.ArrayListUnmanaged(u32) = .empty, - -/// When in debug mode, this tracks if no `finishAir` was missed. -/// Forgetting to call `finishAir` will cause the result to not be -/// stored in our `values` map and therefore cause bugs. -air_bookkeeping: @TypeOf(bookkeeping_init) = bookkeeping_init, - -/// Wasm Value, created when generating an instruction -const WValue = union(enum) { - /// `WValue` which has been freed and may no longer hold - /// any references. - dead: void, - /// May be referenced but is unused - none: void, - /// The value lives on top of the stack - stack: void, - /// Index of the local - local: struct { - /// Contains the index to the local - value: u32, - /// The amount of instructions referencing this `WValue` - references: u32, - }, - /// An immediate 32bit value - imm32: u32, - /// An immediate 64bit value - imm64: u64, - /// Index into the list of simd128 immediates. This `WValue` is - /// only possible in very rare cases, therefore it would be - /// a waste of memory to store the value in a 128 bit integer. - imm128: u32, - /// A constant 32bit float value - float32: f32, - /// A constant 64bit float value - float64: f64, - nav_ref: struct { - nav_index: InternPool.Nav.Index, - offset: i32 = 0, - }, - uav_ref: struct { - ip_index: InternPool.Index, - offset: i32 = 0, - orig_ptr_ty: InternPool.Index = .none, - }, - /// Offset from the bottom of the virtual stack, with the offset - /// pointing to where the value lives. - stack_offset: struct { - /// Contains the actual value of the offset - value: u32, - /// The amount of instructions referencing this `WValue` - references: u32, - }, - - /// Returns the offset from the bottom of the stack. This is useful when - /// we use the load or store instruction to ensure we retrieve the value - /// from the correct position, rather than the value that lives at the - /// bottom of the stack. For instances where `WValue` is not `stack_value` - /// this will return 0, which allows us to simply call this function for all - /// loads and stores without requiring checks everywhere. - fn offset(value: WValue) u32 { - switch (value) { - .stack_offset => |stack_offset| return stack_offset.value, - .dead => unreachable, - else => return 0, - } - } - - /// Promotes a `WValue` to a local when given value is on top of the stack. - /// When encountering a `local` or `stack_offset` this is essentially a no-op. - /// All other tags are illegal. - fn toLocal(value: WValue, gen: *CodeGen, ty: Type) InnerError!WValue { - switch (value) { - .stack => { - const new_local = try gen.allocLocal(ty); - try gen.addLocal(.local_set, new_local.local.value); - return new_local; - }, - .local, .stack_offset => return value, - else => unreachable, - } - } - - /// Marks a local as no longer being referenced and essentially allows - /// us to re-use it somewhere else within the function. - /// The valtype of the local is deducted by using the index of the given `WValue`. - fn free(value: *WValue, gen: *CodeGen) void { - if (value.* != .local) return; - const local_value = value.local.value; - const reserved = gen.args.len + @intFromBool(gen.return_value != .none); - if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals. - - const index = local_value - reserved; - const valtype = gen.mir_locals.items[index]; - switch (valtype) { - .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead - .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return, - .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return, - .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return, - .v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return, - } - log.debug("freed local ({d}) of type {}", .{ local_value, valtype }); - value.* = .dead; - } -}; - -const Op = enum { - @"unreachable", - nop, - block, - loop, - @"if", - @"else", - end, - br, - br_if, - br_table, - @"return", - call, - drop, - select, - global_get, - global_set, - load, - store, - memory_size, - memory_grow, - @"const", - eqz, - eq, - ne, - lt, - gt, - le, - ge, - clz, - ctz, - popcnt, - add, - sub, - mul, - div, - rem, - @"and", - @"or", - xor, - shl, - shr, - rotl, - rotr, - abs, - neg, - ceil, - floor, - trunc, - nearest, - sqrt, - min, - max, - copysign, - wrap, - convert, - demote, - promote, - reinterpret, - extend, -}; - -const OpcodeBuildArguments = struct { - /// First valtype in the opcode (usually represents the type of the output) - valtype1: ?std.wasm.Valtype = null, - /// The operation (e.g. call, unreachable, div, min, sqrt, etc.) - op: Op, - /// Width of the operation (e.g. 8 for i32_load8_s, 16 for i64_extend16_i32_s) - width: ?u8 = null, - /// Second valtype in the opcode name (usually represents the type of the input) - valtype2: ?std.wasm.Valtype = null, - /// Signedness of the op - signedness: ?std.builtin.Signedness = null, -}; - -/// TODO: deprecated, should be split up per tag. -fn buildOpcode(args: OpcodeBuildArguments) std.wasm.Opcode { - switch (args.op) { - .@"unreachable" => unreachable, - .nop => unreachable, - .block => unreachable, - .loop => unreachable, - .@"if" => unreachable, - .@"else" => unreachable, - .end => unreachable, - .br => unreachable, - .br_if => unreachable, - .br_table => unreachable, - .@"return" => unreachable, - .call => unreachable, - .drop => unreachable, - .select => unreachable, - .global_get => unreachable, - .global_set => unreachable, - - .load => if (args.width) |width| switch (width) { - 8 => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_load8_s else return .i32_load8_u, - .i64 => if (args.signedness.? == .signed) return .i64_load8_s else return .i64_load8_u, - .f32, .f64, .v128 => unreachable, - }, - 16 => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_load16_s else return .i32_load16_u, - .i64 => if (args.signedness.? == .signed) return .i64_load16_s else return .i64_load16_u, - .f32, .f64, .v128 => unreachable, - }, - 32 => switch (args.valtype1.?) { - .i64 => if (args.signedness.? == .signed) return .i64_load32_s else return .i64_load32_u, - .i32 => return .i32_load, - .f32 => return .f32_load, - .f64, .v128 => unreachable, - }, - 64 => switch (args.valtype1.?) { - .i64 => return .i64_load, - .f64 => return .f64_load, - else => unreachable, - }, - else => unreachable, - } else switch (args.valtype1.?) { - .i32 => return .i32_load, - .i64 => return .i64_load, - .f32 => return .f32_load, - .f64 => return .f64_load, - .v128 => unreachable, // handled independently - }, - .store => if (args.width) |width| { - switch (width) { - 8 => switch (args.valtype1.?) { - .i32 => return .i32_store8, - .i64 => return .i64_store8, - .f32, .f64, .v128 => unreachable, - }, - 16 => switch (args.valtype1.?) { - .i32 => return .i32_store16, - .i64 => return .i64_store16, - .f32, .f64, .v128 => unreachable, - }, - 32 => switch (args.valtype1.?) { - .i64 => return .i64_store32, - .i32 => return .i32_store, - .f32 => return .f32_store, - .f64, .v128 => unreachable, - }, - 64 => switch (args.valtype1.?) { - .i64 => return .i64_store, - .f64 => return .f64_store, - else => unreachable, - }, - else => unreachable, - } - } else { - switch (args.valtype1.?) { - .i32 => return .i32_store, - .i64 => return .i64_store, - .f32 => return .f32_store, - .f64 => return .f64_store, - .v128 => unreachable, // handled independently - } - }, - - .memory_size => return .memory_size, - .memory_grow => return .memory_grow, - - .@"const" => switch (args.valtype1.?) { - .i32 => return .i32_const, - .i64 => return .i64_const, - .f32 => return .f32_const, - .f64 => return .f64_const, - .v128 => unreachable, // handled independently - }, - - .eqz => switch (args.valtype1.?) { - .i32 => return .i32_eqz, - .i64 => return .i64_eqz, - .f32, .f64, .v128 => unreachable, - }, - .eq => switch (args.valtype1.?) { - .i32 => return .i32_eq, - .i64 => return .i64_eq, - .f32 => return .f32_eq, - .f64 => return .f64_eq, - .v128 => unreachable, // handled independently - }, - .ne => switch (args.valtype1.?) { - .i32 => return .i32_ne, - .i64 => return .i64_ne, - .f32 => return .f32_ne, - .f64 => return .f64_ne, - .v128 => unreachable, // handled independently - }, - - .lt => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_lt_s else return .i32_lt_u, - .i64 => if (args.signedness.? == .signed) return .i64_lt_s else return .i64_lt_u, - .f32 => return .f32_lt, - .f64 => return .f64_lt, - .v128 => unreachable, // handled independently - }, - .gt => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_gt_s else return .i32_gt_u, - .i64 => if (args.signedness.? == .signed) return .i64_gt_s else return .i64_gt_u, - .f32 => return .f32_gt, - .f64 => return .f64_gt, - .v128 => unreachable, // handled independently - }, - .le => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_le_s else return .i32_le_u, - .i64 => if (args.signedness.? == .signed) return .i64_le_s else return .i64_le_u, - .f32 => return .f32_le, - .f64 => return .f64_le, - .v128 => unreachable, // handled independently - }, - .ge => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_ge_s else return .i32_ge_u, - .i64 => if (args.signedness.? == .signed) return .i64_ge_s else return .i64_ge_u, - .f32 => return .f32_ge, - .f64 => return .f64_ge, - .v128 => unreachable, // handled independently - }, - - .clz => switch (args.valtype1.?) { - .i32 => return .i32_clz, - .i64 => return .i64_clz, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .ctz => switch (args.valtype1.?) { - .i32 => return .i32_ctz, - .i64 => return .i64_ctz, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .popcnt => switch (args.valtype1.?) { - .i32 => return .i32_popcnt, - .i64 => return .i64_popcnt, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - - .add => switch (args.valtype1.?) { - .i32 => return .i32_add, - .i64 => return .i64_add, - .f32 => return .f32_add, - .f64 => return .f64_add, - .v128 => unreachable, // handled independently - }, - .sub => switch (args.valtype1.?) { - .i32 => return .i32_sub, - .i64 => return .i64_sub, - .f32 => return .f32_sub, - .f64 => return .f64_sub, - .v128 => unreachable, // handled independently - }, - .mul => switch (args.valtype1.?) { - .i32 => return .i32_mul, - .i64 => return .i64_mul, - .f32 => return .f32_mul, - .f64 => return .f64_mul, - .v128 => unreachable, // handled independently - }, - - .div => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_div_s else return .i32_div_u, - .i64 => if (args.signedness.? == .signed) return .i64_div_s else return .i64_div_u, - .f32 => return .f32_div, - .f64 => return .f64_div, - .v128 => unreachable, // handled independently - }, - .rem => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_rem_s else return .i32_rem_u, - .i64 => if (args.signedness.? == .signed) return .i64_rem_s else return .i64_rem_u, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - - .@"and" => switch (args.valtype1.?) { - .i32 => return .i32_and, - .i64 => return .i64_and, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .@"or" => switch (args.valtype1.?) { - .i32 => return .i32_or, - .i64 => return .i64_or, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .xor => switch (args.valtype1.?) { - .i32 => return .i32_xor, - .i64 => return .i64_xor, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - - .shl => switch (args.valtype1.?) { - .i32 => return .i32_shl, - .i64 => return .i64_shl, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .shr => switch (args.valtype1.?) { - .i32 => if (args.signedness.? == .signed) return .i32_shr_s else return .i32_shr_u, - .i64 => if (args.signedness.? == .signed) return .i64_shr_s else return .i64_shr_u, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .rotl => switch (args.valtype1.?) { - .i32 => return .i32_rotl, - .i64 => return .i64_rotl, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .rotr => switch (args.valtype1.?) { - .i32 => return .i32_rotr, - .i64 => return .i64_rotr, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - - .abs => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_abs, - .f64 => return .f64_abs, - .v128 => unreachable, // handled independently - }, - .neg => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_neg, - .f64 => return .f64_neg, - .v128 => unreachable, // handled independently - }, - .ceil => switch (args.valtype1.?) { - .i64 => unreachable, - .i32 => return .f32_ceil, // when valtype is f16, we store it in i32. - .f32 => return .f32_ceil, - .f64 => return .f64_ceil, - .v128 => unreachable, // handled independently - }, - .floor => switch (args.valtype1.?) { - .i64 => unreachable, - .i32 => return .f32_floor, // when valtype is f16, we store it in i32. - .f32 => return .f32_floor, - .f64 => return .f64_floor, - .v128 => unreachable, // handled independently - }, - .trunc => switch (args.valtype1.?) { - .i32 => if (args.valtype2) |valty| switch (valty) { - .i32 => unreachable, - .i64 => unreachable, - .f32 => if (args.signedness.? == .signed) return .i32_trunc_f32_s else return .i32_trunc_f32_u, - .f64 => if (args.signedness.? == .signed) return .i32_trunc_f64_s else return .i32_trunc_f64_u, - .v128 => unreachable, // handled independently - } else return .f32_trunc, // when no valtype2, it's an f16 instead which is stored in an i32. - .i64 => switch (args.valtype2.?) { - .i32 => unreachable, - .i64 => unreachable, - .f32 => if (args.signedness.? == .signed) return .i64_trunc_f32_s else return .i64_trunc_f32_u, - .f64 => if (args.signedness.? == .signed) return .i64_trunc_f64_s else return .i64_trunc_f64_u, - .v128 => unreachable, // handled independently - }, - .f32 => return .f32_trunc, - .f64 => return .f64_trunc, - .v128 => unreachable, // handled independently - }, - .nearest => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_nearest, - .f64 => return .f64_nearest, - .v128 => unreachable, // handled independently - }, - .sqrt => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_sqrt, - .f64 => return .f64_sqrt, - .v128 => unreachable, // handled independently - }, - .min => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_min, - .f64 => return .f64_min, - .v128 => unreachable, // handled independently - }, - .max => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_max, - .f64 => return .f64_max, - .v128 => unreachable, // handled independently - }, - .copysign => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => return .f32_copysign, - .f64 => return .f64_copysign, - .v128 => unreachable, // handled independently - }, - - .wrap => switch (args.valtype1.?) { - .i32 => switch (args.valtype2.?) { - .i32 => unreachable, - .i64 => return .i32_wrap_i64, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .i64, .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .convert => switch (args.valtype1.?) { - .i32, .i64 => unreachable, - .f32 => switch (args.valtype2.?) { - .i32 => if (args.signedness.? == .signed) return .f32_convert_i32_s else return .f32_convert_i32_u, - .i64 => if (args.signedness.? == .signed) return .f32_convert_i64_s else return .f32_convert_i64_u, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .f64 => switch (args.valtype2.?) { - .i32 => if (args.signedness.? == .signed) return .f64_convert_i32_s else return .f64_convert_i32_u, - .i64 => if (args.signedness.? == .signed) return .f64_convert_i64_s else return .f64_convert_i64_u, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - .v128 => unreachable, // handled independently - }, - .demote => if (args.valtype1.? == .f32 and args.valtype2.? == .f64) return .f32_demote_f64 else unreachable, - .promote => if (args.valtype1.? == .f64 and args.valtype2.? == .f32) return .f64_promote_f32 else unreachable, - .reinterpret => switch (args.valtype1.?) { - .i32 => if (args.valtype2.? == .f32) return .i32_reinterpret_f32 else unreachable, - .i64 => if (args.valtype2.? == .f64) return .i64_reinterpret_f64 else unreachable, - .f32 => if (args.valtype2.? == .i32) return .f32_reinterpret_i32 else unreachable, - .f64 => if (args.valtype2.? == .i64) return .f64_reinterpret_i64 else unreachable, - .v128 => unreachable, // handled independently - }, - .extend => switch (args.valtype1.?) { - .i32 => switch (args.width.?) { - 8 => if (args.signedness.? == .signed) return .i32_extend8_s else unreachable, - 16 => if (args.signedness.? == .signed) return .i32_extend16_s else unreachable, - else => unreachable, - }, - .i64 => switch (args.width.?) { - 8 => if (args.signedness.? == .signed) return .i64_extend8_s else unreachable, - 16 => if (args.signedness.? == .signed) return .i64_extend16_s else unreachable, - 32 => if (args.signedness.? == .signed) return .i64_extend32_s else unreachable, - else => unreachable, - }, - .f32, .f64 => unreachable, - .v128 => unreachable, // handled independently - }, - } -} - -test "Wasm - buildOpcode" { - // Make sure buildOpcode is referenced, and test some examples - const i32_const = buildOpcode(.{ .op = .@"const", .valtype1 = .i32 }); - const i64_extend32_s = buildOpcode(.{ .op = .extend, .valtype1 = .i64, .width = 32, .signedness = .signed }); - const f64_reinterpret_i64 = buildOpcode(.{ .op = .reinterpret, .valtype1 = .f64, .valtype2 = .i64 }); - - try testing.expectEqual(@as(std.wasm.Opcode, .i32_const), i32_const); - try testing.expectEqual(@as(std.wasm.Opcode, .i64_extend32_s), i64_extend32_s); - try testing.expectEqual(@as(std.wasm.Opcode, .f64_reinterpret_i64), f64_reinterpret_i64); -} - -/// Hashmap to store generated `WValue` for each `Air.Inst.Ref` -pub const ValueTable = std.AutoArrayHashMapUnmanaged(Air.Inst.Ref, WValue); - -const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; - -const InnerError = error{ - OutOfMemory, - /// An error occurred when trying to lower AIR to MIR. - CodegenFail, - /// Compiler implementation could not handle a large integer. - Overflow, -} || link.File.UpdateDebugInfoError; - -pub fn deinit(cg: *CodeGen) void { - const gpa = cg.gpa; - for (cg.branches.items) |*branch| branch.deinit(gpa); - cg.branches.deinit(gpa); - cg.blocks.deinit(gpa); - cg.loops.deinit(gpa); - cg.simd_immediates.deinit(gpa); - cg.free_locals_i32.deinit(gpa); - cg.free_locals_i64.deinit(gpa); - cg.free_locals_f32.deinit(gpa); - cg.free_locals_f64.deinit(gpa); - cg.free_locals_v128.deinit(gpa); - cg.mir_instructions.deinit(gpa); - cg.mir_extra.deinit(gpa); - cg.mir_locals.deinit(gpa); - cg.mir_uavs.deinit(gpa); - cg.mir_indirect_function_set.deinit(gpa); - cg.mir_func_tys.deinit(gpa); - cg.* = undefined; -} - -fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { - const zcu = cg.pt.zcu; - const func = zcu.funcInfo(cg.func_index); - return zcu.codegenFail(func.owner_nav, fmt, args); -} - -/// Resolves the `WValue` for the given instruction `inst` -/// When the given instruction has a `Value`, it returns a constant instead -fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue { - var branch_index = cg.branches.items.len; - while (branch_index > 0) : (branch_index -= 1) { - const branch = cg.branches.items[branch_index - 1]; - if (branch.values.get(ref)) |value| { - return value; - } - } - - // when we did not find an existing instruction, it - // means we must generate it from a constant. - // We always store constants in the most outer branch as they must never - // be removed. The most outer branch is always at index 0. - const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref); - assert(!gop.found_existing); - - const pt = cg.pt; - const zcu = pt.zcu; - const val = (try cg.air.value(ref, pt)).?; - const ty = cg.typeOf(ref); - if (!ty.hasRuntimeBitsIgnoreComptime(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) { - gop.value_ptr.* = .none; - return .none; - } - - // When we need to pass the value by reference (such as a struct), we will - // leverage `generateSymbol` to lower the constant to bytes and emit it - // to the 'rodata' section. We then return the index into the section as `WValue`. - // - // In the other cases, we will simply lower the constant to a value that fits - // into a single local (such as a pointer, integer, bool, etc). - const result: WValue = if (isByRef(ty, zcu, cg.target)) - .{ .uav_ref = .{ .ip_index = val.toIntern() } } - else - try cg.lowerConstant(val, ty); - - gop.value_ptr.* = result; - return result; -} - -fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { - const zcu = cg.pt.zcu; - const ty = val.typeOf(zcu); - - return if (isByRef(ty, zcu, cg.target)) - .{ .uav_ref = .{ .ip_index = val.toIntern() } } - else - try cg.lowerConstant(val, ty); -} - -/// NOTE: if result == .stack, it will be stored in .local -fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { - assert(operands.len <= Air.Liveness.bpi - 1); - var tomb_bits = cg.liveness.getTombBits(inst); - for (operands) |operand| { - const dies = @as(u1, @truncate(tomb_bits)) != 0; - tomb_bits >>= 1; - if (!dies) continue; - processDeath(cg, operand); - } - - // results of `none` can never be referenced. - if (result != .none) { - const trackable_result = if (result != .stack) - result - else - try result.toLocal(cg, cg.typeOfIndex(inst)); - const branch = cg.currentBranch(); - branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result); - } - - if (std.debug.runtime_safety) { - cg.air_bookkeeping += 1; - } -} - -const Branch = struct { - values: ValueTable = .{}, - - fn deinit(branch: *Branch, gpa: Allocator) void { - branch.values.deinit(gpa); - branch.* = undefined; - } -}; - -inline fn currentBranch(cg: *CodeGen) *Branch { - return &cg.branches.items[cg.branches.items.len - 1]; -} - -const BigTomb = struct { - gen: *CodeGen, - inst: Air.Inst.Index, - lbt: Air.Liveness.BigTomb, - - fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { - const dies = bt.lbt.feed(); - if (!dies) return; - // This will be a nop for interned constants. - processDeath(bt.gen, op_ref); - } - - fn finishAir(bt: *BigTomb, result: WValue) void { - assert(result != .stack); - if (result != .none) { - bt.gen.currentBranch().values.putAssumeCapacityNoClobber(bt.inst.toRef(), result); - } - - if (std.debug.runtime_safety) { - bt.gen.air_bookkeeping += 1; - } - } -}; - -fn iterateBigTomb(cg: *CodeGen, inst: Air.Inst.Index, operand_count: usize) !BigTomb { - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, operand_count + 1); - return BigTomb{ - .gen = cg, - .inst = inst, - .lbt = cg.liveness.iterateBigTomb(inst), - }; -} - -fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void { - if (ref.toIndex() == null) return; - // Branches are currently only allowed to free locals allocated - // within their own branch. - // TODO: Upon branch consolidation free any locals if needed. - const value = cg.currentBranch().values.getPtr(ref) orelse return; - if (value.* != .local) return; - const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none); - if (value.local.value < reserved_indexes) { - return; // function arguments can never be re-used - } - log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @intFromEnum(ref.toIndex().?), value.local.value }); - value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer - if (value.local.references == 0) { - value.free(cg); - } -} - -fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void { - try cg.mir_instructions.append(cg.gpa, inst); -} - -fn addTag(cg: *CodeGen, tag: Mir.Inst.Tag) error{OutOfMemory}!void { - try cg.addInst(.{ .tag = tag, .data = .{ .tag = {} } }); -} - -fn addExtended(cg: *CodeGen, opcode: std.wasm.MiscOpcode) error{OutOfMemory}!void { - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - try cg.mir_extra.append(cg.gpa, @intFromEnum(opcode)); - try cg.addInst(.{ .tag = .misc_prefix, .data = .{ .payload = extra_index } }); -} - -fn addLabel(cg: *CodeGen, tag: Mir.Inst.Tag, label: u32) error{OutOfMemory}!void { - try cg.addInst(.{ .tag = tag, .data = .{ .label = label } }); -} - -fn addLocal(cg: *CodeGen, tag: Mir.Inst.Tag, local: u32) error{OutOfMemory}!void { - try cg.addInst(.{ .tag = tag, .data = .{ .local = local } }); -} - -/// Accepts an unsigned 32bit integer rather than a signed integer to -/// prevent us from having to bitcast multiple times as most values -/// within codegen are represented as unsigned rather than signed. -fn addImm32(cg: *CodeGen, imm: u32) error{OutOfMemory}!void { - try cg.addInst(.{ .tag = .i32_const, .data = .{ .imm32 = @bitCast(imm) } }); -} - -/// Accepts an unsigned 64bit integer rather than a signed integer to -/// prevent us from having to bitcast multiple times as most values -/// within codegen are represented as unsigned rather than signed. -fn addImm64(cg: *CodeGen, imm: u64) error{OutOfMemory}!void { - const extra_index = try cg.addExtra(Mir.Imm64.init(imm)); - try cg.addInst(.{ .tag = .i64_const, .data = .{ .payload = extra_index } }); -} - -/// Accepts the index into the list of 128bit-immediates -fn addImm128(cg: *CodeGen, index: u32) error{OutOfMemory}!void { - const simd_values = cg.simd_immediates.items[index]; - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - // tag + 128bit value - try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 5); - cg.mir_extra.appendAssumeCapacity(@intFromEnum(std.wasm.SimdOpcode.v128_const)); - cg.mir_extra.appendSliceAssumeCapacity(@alignCast(mem.bytesAsSlice(u32, &simd_values))); - try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); -} - -fn addFloat64(cg: *CodeGen, float: f64) error{OutOfMemory}!void { - const extra_index = try cg.addExtra(Mir.Float64.init(float)); - try cg.addInst(.{ .tag = .f64_const, .data = .{ .payload = extra_index } }); -} - -/// Inserts an instruction to load/store from/to wasm's linear memory dependent on the given `tag`. -fn addMemArg(cg: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOfMemory}!void { - const extra_index = try cg.addExtra(mem_arg); - try cg.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } }); -} - -/// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the -/// given `tag`. -fn addAtomicMemArg(cg: *CodeGen, tag: std.wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void { - const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @intFromEnum(tag) })); - _ = try cg.addExtra(mem_arg); - try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); -} - -/// Helper function to emit atomic mir opcodes. -fn addAtomicTag(cg: *CodeGen, tag: std.wasm.AtomicsOpcode) error{OutOfMemory}!void { - const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @intFromEnum(tag) })); - try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); -} - -/// Appends entries to `mir_extra` based on the type of `extra`. -/// Returns the index into `mir_extra` -fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { - const fields = std.meta.fields(@TypeOf(extra)); - try cg.mir_extra.ensureUnusedCapacity(cg.gpa, fields.len); - return cg.addExtraAssumeCapacity(extra); -} - -/// Appends entries to `mir_extra` based on the type of `extra`. -/// Returns the index into `mir_extra` -fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { - const fields = std.meta.fields(@TypeOf(extra)); - const result: u32 = @intCast(cg.mir_extra.items.len); - inline for (fields) |field| { - cg.mir_extra.appendAssumeCapacity(switch (field.type) { - u32 => @field(extra, field.name), - i32 => @bitCast(@field(extra, field.name)), - InternPool.Index, - InternPool.Nav.Index, - => @intFromEnum(@field(extra, field.name)), - else => |field_type| @compileError("Unsupported field type " ++ @typeName(field_type)), - }); - } - return result; -} - -/// For `std.builtin.CallingConvention.auto`. -pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.Valtype { - const ip = &zcu.intern_pool; - return switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(target)) { - 16 => .i32, // stored/loaded as u16 - 32 => .f32, - 64 => .f64, - 80, 128 => .i32, - else => unreachable, - }, - .int, .@"enum" => switch (ty.intInfo(zcu).bits) { - 0...32 => .i32, - 33...64 => .i64, - else => .i32, - }, - .@"struct" => blk: { - if (zcu.typeToPackedStruct(ty)) |packed_struct| { - const backing_int_ty = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); - break :blk typeToValtype(backing_int_ty, zcu, target); - } else { - break :blk .i32; - } - }, - .vector => switch (CodeGen.determineSimdStoreStrategy(ty, zcu, target)) { - .direct => .v128, - .unrolled => .i32, - }, - .@"union" => switch (ty.containerLayout(zcu)) { - .@"packed" => switch (ty.bitSize(zcu)) { - 0...32 => .i32, - 33...64 => .i64, - else => .i32, - }, - else => .i32, - }, - else => .i32, // all represented as reference/immediate - }; -} - -/// Using a given `Type`, returns the corresponding wasm value type -/// Differently from `typeToValtype` this also allows `void` to create a block -/// with no return type -fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType { - return switch (ty.ip_index) { - .void_type, .noreturn_type => .empty, - else => .fromValtype(typeToValtype(ty, zcu, target)), - }; -} - -/// Writes the bytecode depending on the given `WValue` in `val` -fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { - switch (value) { - .dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?) - .none, .stack => {}, // no-op - .local => |idx| try cg.addLocal(.local_get, idx.value), - .imm32 => |val| try cg.addImm32(val), - .imm64 => |val| try cg.addImm64(val), - .imm128 => |val| try cg.addImm128(val), - .float32 => |val| try cg.addInst(.{ .tag = .f32_const, .data = .{ .float32 = val } }), - .float64 => |val| try cg.addFloat64(val), - .nav_ref => |nav_ref| { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - if (ip.getNav(nav_ref.nav_index).isFn(ip)) { - assert(nav_ref.offset == 0); - try cg.mir_indirect_function_set.put(cg.gpa, nav_ref.nav_index, {}); - try cg.addInst(.{ .tag = .func_ref, .data = .{ .nav_index = nav_ref.nav_index } }); - } else if (nav_ref.offset == 0) { - try cg.addInst(.{ .tag = .nav_ref, .data = .{ .nav_index = nav_ref.nav_index } }); - } else { - try cg.addInst(.{ - .tag = .nav_ref_off, - .data = .{ - .payload = try cg.addExtra(Mir.NavRefOff{ - .nav_index = nav_ref.nav_index, - .offset = nav_ref.offset, - }), - }, - }); - } - }, - .uav_ref => |uav| { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - assert(!ip.isFunctionType(ip.typeOf(uav.ip_index))); - const gop = try cg.mir_uavs.getOrPut(cg.gpa, uav.ip_index); - const this_align: Alignment = a: { - if (uav.orig_ptr_ty == .none) break :a .none; - const ptr_type = ip.indexToKey(uav.orig_ptr_ty).ptr_type; - const this_align = ptr_type.flags.alignment; - if (this_align == .none) break :a .none; - const abi_align = Type.fromInterned(ptr_type.child).abiAlignment(zcu); - if (this_align.compare(.lte, abi_align)) break :a .none; - break :a this_align; - }; - if (!gop.found_existing or - gop.value_ptr.* == .none or - (this_align != .none and this_align.compare(.gt, gop.value_ptr.*))) - { - gop.value_ptr.* = this_align; - } - if (uav.offset == 0) { - try cg.addInst(.{ - .tag = .uav_ref, - .data = .{ .ip_index = uav.ip_index }, - }); - } else { - try cg.addInst(.{ - .tag = .uav_ref_off, - .data = .{ .payload = try cg.addExtra(@as(Mir.UavRefOff, .{ - .value = uav.ip_index, - .offset = uav.offset, - })) }, - }); - } - }, - .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset - } -} - -/// If given a local or stack-offset, increases the reference count by 1. -/// The old `WValue` found at instruction `ref` is then replaced by the -/// modified `WValue` and returned. When given a non-local or non-stack-offset, -/// returns the given `operand` itfunc instead. -fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue { - if (operand != .local and operand != .stack_offset) return operand; - var new_value = operand; - switch (new_value) { - .local => |*local| local.references += 1, - .stack_offset => |*stack_offset| stack_offset.references += 1, - else => unreachable, - } - const old_value = cg.getResolvedInst(ref); - old_value.* = new_value; - return new_value; -} - -/// From a reference, returns its resolved `WValue`. -/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet. -fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { - var index = cg.branches.items.len; - while (index > 0) : (index -= 1) { - const branch = cg.branches.items[index - 1]; - if (branch.values.getPtr(ref)) |value| { - return value; - } - } - unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead. -} - -/// Creates one locals for a given `Type`. -/// Returns a corresponding `Wvalue` with `local` as active tag -fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - const valtype = typeToValtype(ty, zcu, cg.target); - const index_or_null = switch (valtype) { - .i32 => cg.free_locals_i32.pop(), - .i64 => cg.free_locals_i64.pop(), - .f32 => cg.free_locals_f32.pop(), - .f64 => cg.free_locals_f64.pop(), - .v128 => cg.free_locals_v128.pop(), - }; - if (index_or_null) |index| { - log.debug("reusing local ({d}) of type {}", .{ index, valtype }); - return .{ .local = .{ .value = index, .references = 1 } }; - } - log.debug("new local of type {}", .{valtype}); - return cg.ensureAllocLocal(ty); -} - -/// Ensures a new local will be created. This is useful when it's useful -/// to use a zero-initialized local. -fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target)); - const initial_index = cg.local_index; - cg.local_index += 1; - return .{ .local = .{ .value = initial_index, .references = 1 } }; -} - -pub const Error = error{ - OutOfMemory, - /// Compiler was asked to operate on a number larger than supported. - Overflow, - /// Indicates the error is already stored in Zcu `failed_codegen`. - CodegenFail, -}; - -pub fn generate( - bin_file: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - air: *const Air, - liveness: *const ?Air.Liveness, -) Error!Mir { - _ = src_loc; - _ = bin_file; - const zcu = pt.zcu; - const gpa = zcu.gpa; - const cg = zcu.funcInfo(func_index); - const file_scope = zcu.navFileScope(cg.owner_nav); - const target = &file_scope.mod.?.resolved_target.result; - const fn_ty = zcu.navValue(cg.owner_nav).typeOf(zcu); - const fn_info = zcu.typeToFunc(fn_ty).?; - const ret_ty: Type = .fromInterned(fn_info.return_type); - const any_returns = !firstParamSRet(fn_info.cc, ret_ty, zcu, target) and ret_ty.hasRuntimeBitsIgnoreComptime(zcu); - - var cc_result = try resolveCallingConventionValues(zcu, fn_ty, target); - defer cc_result.deinit(gpa); - - var code_gen: CodeGen = .{ - .gpa = gpa, - .pt = pt, - .air = air.*, - .liveness = liveness.*.?, - .owner_nav = cg.owner_nav, - .target = target, - .ptr_size = switch (target.cpu.arch) { - .wasm32 => .wasm32, - .wasm64 => .wasm64, - else => unreachable, - }, - .func_index = func_index, - .args = cc_result.args, - .return_value = cc_result.return_value, - .local_index = cc_result.local_index, - .mir_instructions = .empty, - .mir_extra = .empty, - .mir_locals = .empty, - .mir_uavs = .empty, - .mir_indirect_function_set = .empty, - .mir_func_tys = .empty, - .error_name_table_ref_count = 0, - }; - defer code_gen.deinit(); - - try code_gen.mir_func_tys.putNoClobber(gpa, fn_ty.toIntern(), {}); - - return generateInner(&code_gen, any_returns) catch |err| switch (err) { - error.CodegenFail, - error.OutOfMemory, - error.Overflow, - => |e| return e, - else => |e| return code_gen.fail("failed to generate function: {s}", .{@errorName(e)}), - }; -} - -fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir { - const zcu = cg.pt.zcu; - try cg.branches.append(cg.gpa, .{}); - // clean up outer branch - defer { - var outer_branch = cg.branches.pop().?; - outer_branch.deinit(cg.gpa); - assert(cg.branches.items.len == 0); // missing branch merge - } - // Generate MIR for function body - try cg.genBody(cg.air.getMainBody()); - - // In case we have a return value, but the last instruction is a noreturn (such as a while loop) - // we emit an unreachable instruction to tell the stack validator that part will never be reached. - if (any_returns and cg.air.instructions.len > 0) { - const inst: Air.Inst.Index = @enumFromInt(cg.air.instructions.len - 1); - const last_inst_ty = cg.typeOfIndex(inst); - if (!last_inst_ty.hasRuntimeBitsIgnoreComptime(zcu) or last_inst_ty.isNoReturn(zcu)) { - try cg.addTag(.@"unreachable"); - } - } - // End of function body - try cg.addTag(.end); - try cg.addTag(.dbg_epilogue_begin); - - var mir: Mir = .{ - .instructions = cg.mir_instructions.toOwnedSlice(), - .extra = &.{}, // fallible so assigned after errdefer - .locals = &.{}, // fallible so assigned after errdefer - .prologue = if (cg.initial_stack_value == .none) .none else .{ - .sp_local = cg.initial_stack_value.local.value, - .flags = .{ .stack_alignment = cg.stack_alignment }, - .stack_size = cg.stack_size, - .bottom_stack_local = cg.bottom_stack_value.local.value, - }, - .uavs = cg.mir_uavs.move(), - .indirect_function_set = cg.mir_indirect_function_set.move(), - .func_tys = cg.mir_func_tys.move(), - .error_name_table_ref_count = cg.error_name_table_ref_count, - }; - errdefer mir.deinit(cg.gpa); - mir.extra = try cg.mir_extra.toOwnedSlice(cg.gpa); - mir.locals = try cg.mir_locals.toOwnedSlice(cg.gpa); - return mir; -} - -const CallWValues = struct { - args: []WValue, - return_value: WValue, - local_index: u32, - - fn deinit(values: *CallWValues, gpa: Allocator) void { - gpa.free(values.args); - values.* = undefined; - } -}; - -fn resolveCallingConventionValues( - zcu: *const Zcu, - fn_ty: Type, - target: *const std.Target, -) Allocator.Error!CallWValues { - const gpa = zcu.gpa; - const ip = &zcu.intern_pool; - const fn_info = zcu.typeToFunc(fn_ty).?; - const cc = fn_info.cc; - - var result: CallWValues = .{ - .args = &.{}, - .return_value = .none, - .local_index = 0, - }; - if (cc == .naked) return result; - - var args = std.array_list.Managed(WValue).init(gpa); - defer args.deinit(); - - // Check if we store the result as a pointer to the stack rather than - // by value - if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) { - // the sret arg will be passed as first argument, therefore we - // set the `return_value` before allocating locals for regular args. - result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } }; - result.local_index += 1; - } - - switch (cc) { - .auto => { - for (fn_info.param_types.get(ip)) |ty| { - if (!Type.fromInterned(ty).hasRuntimeBitsIgnoreComptime(zcu)) { - continue; - } - - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); - result.local_index += 1; - } - }, - .wasm_mvp => { - for (fn_info.param_types.get(ip)) |ty| { - if (!Type.fromInterned(ty).hasRuntimeBitsIgnoreComptime(zcu)) { - continue; - } - switch (abi.classifyType(.fromInterned(ty), zcu)) { - .direct => |scalar_ty| if (!abi.lowerAsDoubleI64(scalar_ty, zcu)) { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); - result.local_index += 1; - } else { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); - try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } }); - result.local_index += 2; - }, - .indirect => { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); - result.local_index += 1; - }, - } - } - }, - else => unreachable, // Frontend is responsible for emitting an error earlier. - } - result.args = try args.toOwnedSlice(); - return result; -} - -pub fn firstParamSRet( - cc: std.builtin.CallingConvention, - return_type: Type, - zcu: *const Zcu, - target: *const std.Target, -) bool { - if (!return_type.hasRuntimeBitsIgnoreComptime(zcu)) return false; - switch (cc) { - .@"inline" => unreachable, - .auto => return isByRef(return_type, zcu, target), - .wasm_mvp => switch (abi.classifyType(return_type, zcu)) { - .direct => |scalar_ty| return abi.lowerAsDoubleI64(scalar_ty, zcu), - .indirect => return true, - }, - else => return false, - } -} - -/// Lowers a Zig type and its value based on a given calling convention to ensure -/// it matches the ABI. -fn lowerArg(cg: *CodeGen, cc: std.builtin.CallingConvention, ty: Type, value: WValue) !void { - if (cc != .wasm_mvp) { - return cg.lowerToStack(value); - } - - const zcu = cg.pt.zcu; - - switch (abi.classifyType(ty, zcu)) { - .direct => |scalar_type| if (!abi.lowerAsDoubleI64(scalar_type, zcu)) { - if (!isByRef(ty, zcu, cg.target)) { - return cg.lowerToStack(value); - } else { - switch (value) { - .nav_ref, .stack_offset => _ = try cg.load(value, scalar_type, 0), - .dead => unreachable, - else => try cg.emitWValue(value), - } - } - } else { - assert(ty.abiSize(zcu) == 16); - // in this case we have an integer or float that must be lowered as 2 i64's. - try cg.emitWValue(value); - try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 }); - try cg.emitWValue(value); - try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 }); - }, - .indirect => return cg.lowerToStack(value), - } -} - -/// Lowers a `WValue` to the stack. This means when the `value` results in -/// `.stack_offset` we calculate the pointer of this offset and use that. -/// The value is left on the stack, and not stored in any temporary. -fn lowerToStack(cg: *CodeGen, value: WValue) !void { - switch (value) { - .stack_offset => |offset| { - try cg.emitWValue(value); - if (offset.value > 0) { - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(offset.value); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(offset.value); - try cg.addTag(.i64_add); - }, - } - } - }, - else => try cg.emitWValue(value), - } -} - -/// Creates a local for the initial stack value -/// Asserts `initial_stack_value` is `.none` -fn initializeStack(cg: *CodeGen) !void { - assert(cg.initial_stack_value == .none); - // Reserve a local to store the current stack pointer - // We can later use this local to set the stack pointer back to the value - // we have stored here. - cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize); - // Also reserve a local to store the bottom stack value - cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize); -} - -/// Reads the stack pointer from `Context.initial_stack_value` and writes it -/// to the global stack pointer variable -fn restoreStackPointer(cg: *CodeGen) !void { - // only restore the pointer if it was initialized - if (cg.initial_stack_value == .none) return; - // Get the original stack pointer's value - try cg.emitWValue(cg.initial_stack_value); - - try cg.addTag(.global_set_sp); -} - -/// From a given type, will create space on the virtual stack to store the value of such type. -/// This returns a `WValue` with its active tag set to `local`, containing the index to the local -/// that points to the position on the virtual stack. This function should be used instead of -/// moveStack unless a local was already created to store the pointer. -/// -/// Asserts Type has codegenbits -fn allocStack(cg: *CodeGen, ty: Type) !WValue { - const pt = cg.pt; - const zcu = pt.zcu; - assert(ty.hasRuntimeBitsIgnoreComptime(zcu)); - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - - const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse { - return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{ - ty.fmt(pt), ty.abiSize(zcu), - }); - }; - const abi_align = ty.abiAlignment(zcu); - - cg.stack_alignment = cg.stack_alignment.max(abi_align); - - const offset: u32 = @intCast(abi_align.forward(cg.stack_size)); - defer cg.stack_size = offset + abi_size; - - return .{ .stack_offset = .{ .value = offset, .references = 1 } }; -} - -/// From a given AIR instruction generates a pointer to the stack where -/// the value of its type will live. -/// This is different from allocStack where this will use the pointer's alignment -/// if it is set, to ensure the stack alignment will be set correctly. -fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue { - const pt = cg.pt; - const zcu = pt.zcu; - const ptr_ty = cg.typeOfIndex(inst); - const pointee_ty = ptr_ty.childType(zcu); - - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - - if (!pointee_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return cg.allocStack(Type.usize); // create a value containing just the stack pointer. - } - - const abi_alignment = ptr_ty.ptrAlignment(zcu); - const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse { - return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{ - pointee_ty.fmt(pt), pointee_ty.abiSize(zcu), - }); - }; - cg.stack_alignment = cg.stack_alignment.max(abi_alignment); - - const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size)); - defer cg.stack_size = offset + abi_size; - - return .{ .stack_offset = .{ .value = offset, .references = 1 } }; -} - -/// From given zig bitsize, returns the wasm bitsize -fn toWasmBits(bits: u16) ?u16 { - return for ([_]u16{ 32, 64, 128 }) |wasm_bits| { - if (bits <= wasm_bits) return wasm_bits; - } else null; -} - -/// Performs a copy of bytes for a given type. Copying all bytes -/// from rhs to lhs. -fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { - const len_known_neq_0 = switch (len) { - .imm32 => |val| if (val != 0) true else return, - .imm64 => |val| if (val != 0) true else return, - else => false, - }; - // When bulk_memory is enabled, we lower it to wasm's memcpy instruction. - // If not, we lower it ourselves manually - if (cg.target.cpu.has(.wasm, .bulk_memory)) { - const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0); - const emit_check = !(len0_ok or len_known_neq_0); - - if (emit_check) { - try cg.startBlock(.block, .empty); - - // Even if `len` is zero, the spec requires an implementation to trap if `src + len` or - // `dst + len` are out of memory bounds. This can easily happen in Zig in a case such - // as: - // - // const dst: [*]u8 = undefined; - // const src: [*]u8 = undefined; - // var len: usize = runtime_zero(); - // @memcpy(dst[0..len], src[0..len]); - // - // So explicitly avoid using `memory.copy` in the `len == 0` case. Lovely design. - try cg.emitWValue(len); - try cg.addTag(.i32_eqz); - try cg.addLabel(.br_if, 0); - } - - try cg.lowerToStack(dst); - try cg.lowerToStack(src); - try cg.emitWValue(len); - try cg.addExtended(.memory_copy); - - if (emit_check) { - try cg.endBlock(); - } - - return; - } - - // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having - // the loop during codegen, rather than inserting a runtime loop into the binary. - switch (len) { - .imm32, .imm64 => blk: { - const length = switch (len) { - .imm32 => |val| val, - .imm64 => |val| val, - else => unreachable, - }; - // if the size (length) is more than 32 bytes, we use a runtime loop instead to prevent - // binary size bloat. - if (length > 32) break :blk; - var offset: u32 = 0; - const lhs_base = dst.offset(); - const rhs_base = src.offset(); - while (offset < length) : (offset += 1) { - // get dst's address to store the result - try cg.emitWValue(dst); - // load byte from src's address - try cg.emitWValue(src); - switch (cg.ptr_size) { - .wasm32 => { - try cg.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 }); - try cg.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 }); - }, - .wasm64 => { - try cg.addMemArg(.i64_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 }); - try cg.addMemArg(.i64_store8, .{ .offset = lhs_base + offset, .alignment = 1 }); - }, - } - } - return; - }, - else => {}, - } - - // allocate a local for the offset, and set it to 0. - // This to ensure that inside loops we correctly re-set the counter. - var offset = try cg.allocLocal(Type.usize); // local for counter - defer offset.free(cg); - switch (cg.ptr_size) { - .wasm32 => try cg.addImm32(0), - .wasm64 => try cg.addImm64(0), - } - try cg.addLocal(.local_set, offset.local.value); - - // outer block to jump to when loop is done - try cg.startBlock(.block, .empty); - try cg.startBlock(.loop, .empty); - - // loop condition (offset == length -> break) - { - try cg.emitWValue(offset); - try cg.emitWValue(len); - switch (cg.ptr_size) { - .wasm32 => try cg.addTag(.i32_eq), - .wasm64 => try cg.addTag(.i64_eq), - } - try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished) - } - - // get dst ptr - { - try cg.emitWValue(dst); - try cg.emitWValue(offset); - switch (cg.ptr_size) { - .wasm32 => try cg.addTag(.i32_add), - .wasm64 => try cg.addTag(.i64_add), - } - } - - // get src value and also store in dst - { - try cg.emitWValue(src); - try cg.emitWValue(offset); - switch (cg.ptr_size) { - .wasm32 => { - try cg.addTag(.i32_add); - try cg.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 }); - try cg.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 }); - }, - .wasm64 => { - try cg.addTag(.i64_add); - try cg.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 }); - try cg.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 }); - }, - } - } - - // increment loop counter - { - try cg.emitWValue(offset); - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(1); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(1); - try cg.addTag(.i64_add); - }, - } - try cg.addLocal(.local_set, offset.local.value); - try cg.addLabel(.br, 0); // jump to start of loop - } - try cg.endBlock(); // close off loop block - try cg.endBlock(); // close off outer block -} - -fn ptrSize(cg: *const CodeGen) u16 { - return @divExact(cg.target.ptrBitWidth(), 8); -} - -/// For a given `Type`, will return true when the type will be passed -/// by reference, rather than by value -fn isByRef(ty: Type, zcu: *const Zcu, target: *const std.Target) bool { - const ip = &zcu.intern_pool; - switch (ty.zigTypeTag(zcu)) { - .type, - .comptime_int, - .comptime_float, - .enum_literal, - .undefined, - .null, - .@"opaque", - => unreachable, - - .noreturn, - .void, - .bool, - .error_set, - .@"fn", - .@"anyframe", - => return false, - - .array, - .frame, - => return ty.hasRuntimeBitsIgnoreComptime(zcu), - .@"union" => { - if (zcu.typeToUnion(ty)) |union_obj| { - if (union_obj.flagsUnordered(ip).layout == .@"packed") { - return ty.abiSize(zcu) > 8; - } - } - return ty.hasRuntimeBitsIgnoreComptime(zcu); - }, - .@"struct" => { - if (zcu.typeToPackedStruct(ty)) |packed_struct| { - return isByRef(Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)), zcu, target); - } - return ty.hasRuntimeBitsIgnoreComptime(zcu); - }, - .vector => return determineSimdStoreStrategy(ty, zcu, target) == .unrolled, - .int => return ty.intInfo(zcu).bits > 64, - .@"enum" => return ty.intInfo(zcu).bits > 64, - .float => return ty.floatBits(target) > 64, - .error_union => { - const pl_ty = ty.errorUnionPayload(zcu); - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return false; - } - return true; - }, - .optional => { - if (ty.isPtrLikeOptional(zcu)) return false; - const pl_type = ty.optionalChild(zcu); - if (pl_type.zigTypeTag(zcu) == .error_set) return false; - return pl_type.hasRuntimeBitsIgnoreComptime(zcu); - }, - .pointer => { - // Slices act like struct and will be passed by reference - if (ty.isSlice(zcu)) return true; - return false; - }, - } -} - -const SimdStoreStrategy = enum { - direct, - unrolled, -}; - -/// For a given vector type, returns the `SimdStoreStrategy`. -/// This means when a given type is 128 bits and either the simd128 or relaxed-simd -/// features are enabled, the function will return `.direct`. This would allow to store -/// it using a instruction, rather than an unrolled version. -pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.Target) SimdStoreStrategy { - assert(ty.zigTypeTag(zcu) == .vector); - if (ty.bitSize(zcu) != 128) return .unrolled; - if (target.cpu.has(.wasm, .relaxed_simd) or target.cpu.has(.wasm, .simd128)) { - return .direct; - } - return .unrolled; -} - -/// Creates a new local for a pointer that points to memory with given offset. -/// This can be used to get a pointer to a struct field, error payload, etc. -/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new -/// local value to store the pointer. This allows for local re-use and improves binary size. -fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue { - // do not perform arithmetic when offset is 0. - if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value; - const result_ptr: WValue = switch (action) { - .new => try cg.ensureAllocLocal(Type.usize), - .modify => ptr_value, - }; - try cg.emitWValue(ptr_value); - if (offset + ptr_value.offset() > 0) { - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(@intCast(offset + ptr_value.offset())); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(offset + ptr_value.offset()); - try cg.addTag(.i64_add); - }, - } - } - try cg.addLocal(.local_set, result_ptr.local.value); - return result_ptr; -} - -fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const air_tags = cg.air.instructions.items(.tag); - return switch (air_tags[@intFromEnum(inst)]) { - .inferred_alloc, .inferred_alloc_comptime => unreachable, - - .add => cg.airBinOp(inst, .add), - .add_sat => cg.airSatBinOp(inst, .add), - .add_wrap => cg.airWrapBinOp(inst, .add), - .sub => cg.airBinOp(inst, .sub), - .sub_sat => cg.airSatBinOp(inst, .sub), - .sub_wrap => cg.airWrapBinOp(inst, .sub), - .mul => cg.airBinOp(inst, .mul), - .mul_sat => cg.airSatMul(inst), - .mul_wrap => cg.airWrapBinOp(inst, .mul), - .div_float, .div_exact => cg.airDiv(inst), - .div_trunc => cg.airDivTrunc(inst), - .div_floor => cg.airDivFloor(inst), - .bit_and => cg.airBinOp(inst, .@"and"), - .bit_or => cg.airBinOp(inst, .@"or"), - .bool_and => cg.airBinOp(inst, .@"and"), - .bool_or => cg.airBinOp(inst, .@"or"), - .rem => cg.airRem(inst), - .mod => cg.airMod(inst), - .shl => cg.airWrapBinOp(inst, .shl), - .shl_exact => cg.airBinOp(inst, .shl), - .shl_sat => cg.airShlSat(inst), - .shr, .shr_exact => cg.airBinOp(inst, .shr), - .xor => cg.airBinOp(inst, .xor), - .max => cg.airMaxMin(inst, .fmax, .gt), - .min => cg.airMaxMin(inst, .fmin, .lt), - .mul_add => cg.airMulAdd(inst), - - .sqrt => cg.airUnaryFloatOp(inst, .sqrt), - .sin => cg.airUnaryFloatOp(inst, .sin), - .cos => cg.airUnaryFloatOp(inst, .cos), - .tan => cg.airUnaryFloatOp(inst, .tan), - .exp => cg.airUnaryFloatOp(inst, .exp), - .exp2 => cg.airUnaryFloatOp(inst, .exp2), - .log => cg.airUnaryFloatOp(inst, .log), - .log2 => cg.airUnaryFloatOp(inst, .log2), - .log10 => cg.airUnaryFloatOp(inst, .log10), - .floor => cg.airUnaryFloatOp(inst, .floor), - .ceil => cg.airUnaryFloatOp(inst, .ceil), - .round => cg.airUnaryFloatOp(inst, .round), - .trunc_float => cg.airUnaryFloatOp(inst, .trunc), - .neg => cg.airUnaryFloatOp(inst, .neg), - - .abs => cg.airAbs(inst), - - .add_with_overflow => cg.airAddSubWithOverflow(inst, .add), - .sub_with_overflow => cg.airAddSubWithOverflow(inst, .sub), - .shl_with_overflow => cg.airShlWithOverflow(inst), - .mul_with_overflow => cg.airMulWithOverflow(inst), - - .clz => cg.airClz(inst), - .ctz => cg.airCtz(inst), - - .cmp_eq => cg.airCmp(inst, .eq), - .cmp_gte => cg.airCmp(inst, .gte), - .cmp_gt => cg.airCmp(inst, .gt), - .cmp_lte => cg.airCmp(inst, .lte), - .cmp_lt => cg.airCmp(inst, .lt), - .cmp_neq => cg.airCmp(inst, .neq), - - .cmp_vector => cg.airCmpVector(inst), - .cmp_lt_errors_len => cg.airCmpLtErrorsLen(inst), - - .array_elem_val => cg.airArrayElemVal(inst), - .array_to_slice => cg.airArrayToSlice(inst), - .alloc => cg.airAlloc(inst), - .arg => cg.airArg(inst), - .bitcast => cg.airBitcast(inst), - .block => cg.airBlock(inst), - .trap => cg.airTrap(inst), - .breakpoint => cg.airBreakpoint(inst), - .br => cg.airBr(inst), - .repeat => cg.airRepeat(inst), - .switch_dispatch => cg.airSwitchDispatch(inst), - .cond_br => cg.airCondBr(inst), - .intcast => cg.airIntcast(inst), - .fptrunc => cg.airFptrunc(inst), - .fpext => cg.airFpext(inst), - .int_from_float => cg.airIntFromFloat(inst), - .float_from_int => cg.airFloatFromInt(inst), - .get_union_tag => cg.airGetUnionTag(inst), - - .@"try" => cg.airTry(inst), - .try_cold => cg.airTry(inst), - .try_ptr => cg.airTryPtr(inst), - .try_ptr_cold => cg.airTryPtr(inst), - - .dbg_stmt => cg.airDbgStmt(inst), - .dbg_empty_stmt => try cg.finishAir(inst, .none, &.{}), - .dbg_inline_block => cg.airDbgInlineBlock(inst), - .dbg_var_ptr => cg.airDbgVar(inst, .local_var, true), - .dbg_var_val => cg.airDbgVar(inst, .local_var, false), - .dbg_arg_inline => cg.airDbgVar(inst, .arg, false), - - .call => cg.airCall(inst, .auto), - .call_always_tail => cg.airCall(inst, .always_tail), - .call_never_tail => cg.airCall(inst, .never_tail), - .call_never_inline => cg.airCall(inst, .never_inline), - - .is_err => cg.airIsErr(inst, .i32_ne, .value), - .is_non_err => cg.airIsErr(inst, .i32_eq, .value), - .is_err_ptr => cg.airIsErr(inst, .i32_ne, .ptr), - .is_non_err_ptr => cg.airIsErr(inst, .i32_eq, .ptr), - - .is_null => cg.airIsNull(inst, .i32_eq, .value), - .is_non_null => cg.airIsNull(inst, .i32_ne, .value), - .is_null_ptr => cg.airIsNull(inst, .i32_eq, .ptr), - .is_non_null_ptr => cg.airIsNull(inst, .i32_ne, .ptr), - - .load => cg.airLoad(inst), - .loop => cg.airLoop(inst), - .memset => cg.airMemset(inst, false), - .memset_safe => cg.airMemset(inst, true), - .not => cg.airNot(inst), - .optional_payload => cg.airOptionalPayload(inst), - .optional_payload_ptr => cg.airOptionalPayloadPtr(inst), - .optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst), - .ptr_add => cg.airPtrBinOp(inst, .add), - .ptr_sub => cg.airPtrBinOp(inst, .sub), - .ptr_elem_ptr => cg.airPtrElemPtr(inst), - .ptr_elem_val => cg.airPtrElemVal(inst), - .ret => cg.airRet(inst), - .ret_safe => cg.airRet(inst), // TODO - .ret_ptr => cg.airRetPtr(inst), - .ret_load => cg.airRetLoad(inst), - .splat => cg.airSplat(inst), - .select => cg.airSelect(inst), - .shuffle_one => cg.airShuffleOne(inst), - .shuffle_two => cg.airShuffleTwo(inst), - .reduce => cg.airReduce(inst), - .aggregate_init => cg.airAggregateInit(inst), - .union_init => cg.airUnionInit(inst), - .prefetch => cg.airPrefetch(inst), - .popcount => cg.airPopcount(inst), - .byte_swap => cg.airByteSwap(inst), - .bit_reverse => cg.airBitReverse(inst), - - .slice => cg.airSlice(inst), - .slice_len => cg.airSliceLen(inst), - .slice_elem_val => cg.airSliceElemVal(inst), - .slice_elem_ptr => cg.airSliceElemPtr(inst), - .slice_ptr => cg.airSlicePtr(inst), - .ptr_slice_len_ptr => cg.airPtrSliceFieldPtr(inst, cg.ptrSize()), - .ptr_slice_ptr_ptr => cg.airPtrSliceFieldPtr(inst, 0), - .store => cg.airStore(inst, false), - .store_safe => cg.airStore(inst, true), - - .set_union_tag => cg.airSetUnionTag(inst), - .struct_field_ptr => cg.airStructFieldPtr(inst), - .struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0), - .struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1), - .struct_field_ptr_index_2 => cg.airStructFieldPtrIndex(inst, 2), - .struct_field_ptr_index_3 => cg.airStructFieldPtrIndex(inst, 3), - .struct_field_val => cg.airStructFieldVal(inst), - .field_parent_ptr => cg.airFieldParentPtr(inst), - - .switch_br => cg.airSwitchBr(inst, false), - .loop_switch_br => cg.airSwitchBr(inst, true), - .trunc => cg.airTrunc(inst), - .unreach => cg.airUnreachable(inst), - - .wrap_optional => cg.airWrapOptional(inst), - .unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false), - .unwrap_errunion_payload_ptr => cg.airUnwrapErrUnionPayload(inst, true), - .unwrap_errunion_err => cg.airUnwrapErrUnionError(inst, false), - .unwrap_errunion_err_ptr => cg.airUnwrapErrUnionError(inst, true), - .wrap_errunion_payload => cg.airWrapErrUnionPayload(inst), - .wrap_errunion_err => cg.airWrapErrUnionErr(inst), - .errunion_payload_ptr_set => cg.airErrUnionPayloadPtrSet(inst), - .error_name => cg.airErrorName(inst), - - .wasm_memory_size => cg.airWasmMemorySize(inst), - .wasm_memory_grow => cg.airWasmMemoryGrow(inst), - - .memcpy, .memmove => cg.airMemcpy(inst), - - .ret_addr => cg.airRetAddr(inst), - .tag_name => cg.airTagName(inst), - - .error_set_has_value => cg.airErrorSetHasValue(inst), - .frame_addr => cg.airFrameAddress(inst), - - .runtime_nav_ptr => cg.airRuntimeNavPtr(inst), - - .assembly, - - .err_return_trace, - .set_err_return_trace, - .save_err_return_trace_index, - .is_named_enum_value, - .addrspace_cast, - .vector_store_elem, - .c_va_arg, - .c_va_copy, - .c_va_end, - .c_va_start, - => |tag| return cg.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), - - .atomic_load => cg.airAtomicLoad(inst), - .atomic_store_unordered, - .atomic_store_monotonic, - .atomic_store_release, - .atomic_store_seq_cst, - // in WebAssembly, all atomic instructions are sequentially ordered. - => cg.airAtomicStore(inst), - .atomic_rmw => cg.airAtomicRmw(inst), - .cmpxchg_weak => cg.airCmpxchg(inst), - .cmpxchg_strong => cg.airCmpxchg(inst), - - .add_optimized, - .sub_optimized, - .mul_optimized, - .div_float_optimized, - .div_trunc_optimized, - .div_floor_optimized, - .div_exact_optimized, - .rem_optimized, - .mod_optimized, - .neg_optimized, - .cmp_lt_optimized, - .cmp_lte_optimized, - .cmp_eq_optimized, - .cmp_gte_optimized, - .cmp_gt_optimized, - .cmp_neq_optimized, - .cmp_vector_optimized, - .reduce_optimized, - .int_from_float_optimized, - => return cg.fail("TODO implement optimized float mode", .{}), - - .add_safe, - .sub_safe, - .mul_safe, - .intcast_safe, - .int_from_float_safe, - .int_from_float_optimized_safe, - => return cg.fail("TODO implement safety_checked_instructions", .{}), - - .work_item_id, - .work_group_size, - .work_group_id, - => unreachable, - }; -} - -fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - - for (body) |inst| { - if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) { - continue; - } - const old_bookkeeping_value = cg.air_bookkeeping; - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Air.Liveness.bpi); - try cg.genInst(inst); - - if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) { - std.debug.panic("Missing call to `finishAir` in AIR instruction %{d} ('{t}')", .{ - inst, - cg.air.instructions.items(.tag)[@intFromEnum(inst)], - }); - } - } -} - -fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?; - const ret_ty = Type.fromInterned(fn_info.return_type); - - // result must be stored in the stack and we return a pointer - // to the stack instead - if (cg.return_value != .none) { - try cg.store(cg.return_value, operand, ret_ty, 0); - } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - switch (abi.classifyType(ret_ty, zcu)) { - .direct => |scalar_type| { - assert(!abi.lowerAsDoubleI64(scalar_type, zcu)); - if (!isByRef(ret_ty, zcu, cg.target)) { - try cg.emitWValue(operand); - } else { - _ = try cg.load(operand, scalar_type, 0); - } - }, - .indirect => unreachable, - } - } else { - if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and ret_ty.isError(zcu)) { - try cg.addImm32(0); - } else { - try cg.emitWValue(operand); - } - } - try cg.restoreStackPointer(); - try cg.addTag(.@"return"); - - return cg.finishAir(inst, .none, &.{un_op}); -} - -fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const child_type = cg.typeOfIndex(inst).childType(zcu); - - const result = result: { - if (!child_type.isFnOrHasRuntimeBitsIgnoreComptime(zcu)) { - break :result try cg.allocStack(Type.usize); // create pointer to void - } - - const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?; - if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) { - break :result cg.return_value; - } - - break :result try cg.allocStackPtr(inst); - }; - - return cg.finishAir(inst, result, &.{}); -} - -fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - const ret_ty = cg.typeOf(un_op).childType(zcu); - - const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?; - if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - if (ret_ty.isError(zcu)) { - try cg.addImm32(0); - } - } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) { - // leave on the stack - _ = try cg.load(operand, ret_ty, 0); - } - - try cg.restoreStackPointer(); - try cg.addTag(.@"return"); - return cg.finishAir(inst, .none, &.{un_op}); -} - -fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) InnerError!void { - if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{}); - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = cg.air.extraData(Air.Call, pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]); - const ty = cg.typeOf(pl_op.operand); - - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const fn_ty = switch (ty.zigTypeTag(zcu)) { - .@"fn" => ty, - .pointer => ty.childType(zcu), - else => unreachable, - }; - const ret_ty = fn_ty.fnReturnType(zcu); - const fn_info = zcu.typeToFunc(fn_ty).?; - const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target); - - const callee: ?InternPool.Nav.Index = blk: { - const func_val = (try cg.air.value(pl_op.operand, pt)) orelse break :blk null; - - switch (ip.indexToKey(func_val.toIntern())) { - inline .func, .@"extern" => |x| break :blk x.owner_nav, - .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { - .nav => |nav| break :blk nav, - else => {}, - }, - else => {}, - } - return cg.fail("unable to lower callee to a function index", .{}); - }; - - const sret: WValue = if (first_param_sret) blk: { - const sret_local = try cg.allocStack(ret_ty); - try cg.lowerToStack(sret_local); - break :blk sret_local; - } else .none; - - for (args) |arg| { - const arg_val = try cg.resolveInst(arg); - - const arg_ty = cg.typeOf(arg); - if (!arg_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - - try cg.lowerArg(zcu.typeToFunc(fn_ty).?.cc, arg_ty, arg_val); - } - - if (callee) |nav_index| { - try cg.addInst(.{ .tag = .call_nav, .data = .{ .nav_index = nav_index } }); - } else { - // in this case we call a function pointer - // so load its value onto the stack - assert(ty.zigTypeTag(zcu) == .pointer); - const operand = try cg.resolveInst(pl_op.operand); - try cg.emitWValue(operand); - - try cg.mir_func_tys.put(cg.gpa, fn_ty.toIntern(), {}); - try cg.addInst(.{ - .tag = .call_indirect, - .data = .{ .ip_index = fn_ty.toIntern() }, - }); - } - - const result_value = result_value: { - if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and !ret_ty.isError(zcu)) { - break :result_value .none; - } else if (ret_ty.isNoReturn(zcu)) { - try cg.addTag(.@"unreachable"); - break :result_value .none; - } else if (first_param_sret) { - break :result_value sret; - } else if (zcu.typeToFunc(fn_ty).?.cc == .wasm_mvp) { - switch (abi.classifyType(ret_ty, zcu)) { - .direct => |scalar_type| { - assert(!abi.lowerAsDoubleI64(scalar_type, zcu)); - if (!isByRef(ret_ty, zcu, cg.target)) { - const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); - break :result_value result_local; - } else { - const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); - const result = try cg.allocStack(ret_ty); - try cg.store(result, result_local, scalar_type, 0); - break :result_value result; - } - }, - .indirect => unreachable, - } - } else { - const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); - break :result_value result_local; - } - }; - - var bt = try cg.iterateBigTomb(inst, 1 + args.len); - bt.feed(pl_op.operand); - for (args) |arg| bt.feed(arg); - return bt.finishAir(result_value); -} - -fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const value = try cg.allocStackPtr(inst); - return cg.finishAir(inst, value, &.{}); -} - -fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const ptr_ty = cg.typeOf(bin_op.lhs); - const ptr_info = ptr_ty.ptrInfo(zcu); - const ty = ptr_ty.childType(zcu); - - if (!safety and bin_op.rhs == .undef) { - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); - } - - if (ptr_info.packed_offset.host_size == 0) { - try cg.store(lhs, rhs, ty, 0); - } else { - // at this point we have a non-natural alignment, we must - // load the value, and then shift+or the rhs into the result location. - const host_size = ptr_info.packed_offset.host_size * 8; - const host_ty = try pt.intType(.unsigned, host_size); - const bit_size: u16 = @intCast(ty.bitSize(zcu)); - const bit_offset = ptr_info.packed_offset.bit_offset; - - const mask_val = try cg.resolveValue(val: { - const limbs = try cg.gpa.alloc( - std.math.big.Limb, - std.math.big.int.calcTwosCompLimbCount(host_size) + 1, - ); - defer cg.gpa.free(limbs); - - var mask_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - mask_bigint.setTwosCompIntLimit(.max, .unsigned, host_size); - - if (bit_size != host_size) { - mask_bigint.shiftRight(mask_bigint.toConst(), host_size - bit_size); - } - if (bit_offset != 0) { - mask_bigint.shiftLeft(mask_bigint.toConst(), bit_offset); - } - mask_bigint.bitNotWrap(mask_bigint.toConst(), .unsigned, host_size); - - break :val try pt.intValue_big(host_ty, mask_bigint.toConst()); - }); - - const shift_val: WValue = if (33 <= host_size and host_size <= 64) - .{ .imm64 = bit_offset } - else - .{ .imm32 = bit_offset }; - - if (host_size <= 64) { - try cg.emitWValue(lhs); - } - const loaded = if (host_size <= 64) - try cg.load(lhs, host_ty, 0) - else - lhs; - const anded = try cg.binOp(loaded, mask_val, host_ty, .@"and"); - const extended_value = try cg.intcast(rhs, ty, host_ty); - const shifted_value = if (bit_offset > 0) - try cg.binOp(extended_value, shift_val, host_ty, .shl) - else - extended_value; - const result = try cg.binOp(anded, shifted_value, host_ty, .@"or"); - if (host_size <= 64) { - try cg.store(.stack, result, host_ty, lhs.offset()); - } else { - try cg.store(lhs, result, host_ty, lhs.offset()); - } - } - - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void { - assert(!(lhs != .stack and rhs == .stack)); - const pt = cg.pt; - const zcu = pt.zcu; - const abi_size = ty.abiSize(zcu); - - if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return; - - switch (ty.zigTypeTag(zcu)) { - .error_union => { - const pl_ty = ty.errorUnionPayload(zcu); - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return cg.store(lhs, rhs, Type.anyerror, offset); - } - - const len = @as(u32, @intCast(abi_size)); - assert(offset == 0); - return cg.memcpy(lhs, rhs, .{ .imm32 = len }); - }, - .optional => { - if (ty.isPtrLikeOptional(zcu)) { - return cg.store(lhs, rhs, Type.usize, offset); - } - const pl_ty = ty.optionalChild(zcu); - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return cg.store(lhs, rhs, Type.u8, offset); - } - if (pl_ty.zigTypeTag(zcu) == .error_set) { - return cg.store(lhs, rhs, Type.anyerror, offset); - } - - const len = @as(u32, @intCast(abi_size)); - assert(offset == 0); - return cg.memcpy(lhs, rhs, .{ .imm32 = len }); - }, - .@"struct", .array, .@"union" => if (isByRef(ty, zcu, cg.target)) { - const len = @as(u32, @intCast(abi_size)); - assert(offset == 0); - return cg.memcpy(lhs, rhs, .{ .imm32 = len }); - }, - .vector => switch (determineSimdStoreStrategy(ty, zcu, cg.target)) { - .unrolled => { - const len: u32 = @intCast(abi_size); - return cg.memcpy(lhs, rhs, .{ .imm32 = len }); - }, - .direct => { - try cg.emitWValue(lhs); - try cg.lowerToStack(rhs); - // TODO: Add helper functions for simd opcodes - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - // stores as := opcode, offset, alignment (opcode::memarg) - try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ - @intFromEnum(std.wasm.SimdOpcode.v128_store), - offset + lhs.offset(), - @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0), - }); - return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); - }, - }, - .pointer => { - if (ty.isSlice(zcu)) { - assert(offset == 0); - // store pointer first - // lower it to the stack so we do not have to store rhs into a local first - try cg.emitWValue(lhs); - const ptr_local = try cg.load(rhs, Type.usize, 0); - try cg.store(.stack, ptr_local, Type.usize, 0 + lhs.offset()); - - // retrieve length from rhs, and store that alongside lhs as well - try cg.emitWValue(lhs); - const len_local = try cg.load(rhs, Type.usize, cg.ptrSize()); - try cg.store(.stack, len_local, Type.usize, cg.ptrSize() + lhs.offset()); - return; - } - }, - .int, .@"enum", .float => if (abi_size > 8 and abi_size <= 16) { - assert(offset == 0); - try cg.emitWValue(lhs); - const lsb = try cg.load(rhs, Type.u64, 0); - try cg.store(.stack, lsb, Type.u64, 0 + lhs.offset()); - - try cg.emitWValue(lhs); - const msb = try cg.load(rhs, Type.u64, 8); - try cg.store(.stack, msb, Type.u64, 8 + lhs.offset()); - return; - } else if (abi_size > 16) { - assert(offset == 0); - try cg.memcpy(lhs, rhs, .{ .imm32 = @as(u32, @intCast(ty.abiSize(zcu))) }); - }, - else => if (abi_size > 8) { - return cg.fail("TODO: `store` for type `{f}` with abisize `{d}`", .{ ty.fmt(pt), abi_size }); - }, - } - try cg.emitWValue(lhs); - // In this case we're actually interested in storing the stack position - // into lhs, so we calculate that and emit that instead - try cg.lowerToStack(rhs); - - const valtype = typeToValtype(ty, zcu, cg.target); - const opcode = buildOpcode(.{ - .valtype1 = valtype, - .width = @as(u8, @intCast(abi_size * 8)), - .op = .store, - }); - - // store rhs value at stack pointer's location in memory - try cg.addMemArg( - Mir.Inst.Tag.fromOpcode(opcode), - .{ - .offset = offset + lhs.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }, - ); -} - -fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - const ty = ty_op.ty.toType(); - const ptr_ty = cg.typeOf(ty_op.operand); - const ptr_info = ptr_ty.ptrInfo(zcu); - - if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand}); - - const result = result: { - if (isByRef(ty, zcu, cg.target)) { - const new_local = try cg.allocStack(ty); - try cg.store(new_local, operand, ty, 0); - break :result new_local; - } - - if (ptr_info.packed_offset.host_size == 0) { - const loaded = try cg.load(operand, ty, 0); - const ty_size = ty.abiSize(zcu); - if (ty.isAbiInt(zcu) and ty_size * 8 > ty.bitSize(zcu)) { - const int_elem_ty = try pt.intType(.unsigned, @intCast(ty_size * 8)); - break :result try cg.trunc(loaded, ty, int_elem_ty); - } else { - break :result loaded; - } - } else { - const int_elem_ty = try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8); - const shift_val: WValue = if (ptr_info.packed_offset.host_size <= 4) - .{ .imm32 = ptr_info.packed_offset.bit_offset } - else if (ptr_info.packed_offset.host_size <= 8) - .{ .imm64 = ptr_info.packed_offset.bit_offset } - else - .{ .imm32 = ptr_info.packed_offset.bit_offset }; - - const stack_loaded = if (ptr_info.packed_offset.host_size <= 8) - try cg.load(operand, int_elem_ty, 0) - else - operand; - const shifted = try cg.binOp(stack_loaded, shift_val, int_elem_ty, .shr); - break :result try cg.trunc(shifted, ty, int_elem_ty); - } - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// Loads an operand from the linear memory section. -/// NOTE: Leaves the value on the stack. -fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue { - const zcu = cg.pt.zcu; - // load local's value from memory by its stack position - try cg.emitWValue(operand); - - if (ty.zigTypeTag(zcu) == .vector) { - // TODO: Add helper functions for simd opcodes - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - // stores as := opcode, offset, alignment (opcode::memarg) - try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ - @intFromEnum(std.wasm.SimdOpcode.v128_load), - offset + operand.offset(), - @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }); - try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); - return .stack; - } - - const abi_size: u8 = @intCast(ty.abiSize(zcu)); - const opcode = buildOpcode(.{ - .valtype1 = typeToValtype(ty, zcu, cg.target), - .width = abi_size * 8, - .op = .load, - .signedness = if (ty.isSignedInt(zcu)) .signed else .unsigned, - }); - - try cg.addMemArg( - Mir.Inst.Tag.fromOpcode(opcode), - .{ - .offset = offset + operand.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }, - ); - - return .stack; -} - -fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const arg_index = cg.arg_index; - const arg = cg.args[arg_index]; - const cc = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?.cc; - const arg_ty = cg.typeOfIndex(inst); - if (cc == .wasm_mvp) { - switch (abi.classifyType(arg_ty, zcu)) { - .direct => |scalar_ty| if (!abi.lowerAsDoubleI64(scalar_ty, zcu)) { - cg.arg_index += 1; - } else { - cg.arg_index += 2; - const result = try cg.allocStack(arg_ty); - try cg.store(result, arg, Type.u64, 0); - try cg.store(result, cg.args[arg_index + 1], Type.u64, 8); - return cg.finishAir(inst, result, &.{}); - }, - .indirect => cg.arg_index += 1, - } - } else { - cg.arg_index += 1; - } - - return cg.finishAir(inst, arg, &.{}); -} - -fn airBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const lhs_ty = cg.typeOf(bin_op.lhs); - const rhs_ty = cg.typeOf(bin_op.rhs); - - // For certain operations, such as shifting, the types are different. - // When converting this to a WebAssembly type, they *must* match to perform - // an operation. For this reason we verify if the WebAssembly type is different, in which - // case we first coerce the operands to the same type before performing the operation. - // For big integers we can ignore this as we will call into compiler-rt which handles this. - const result = switch (op) { - .shr, .shl => result: { - if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) { - return cg.fail("TODO: implement vector '{s}' with scalar rhs", .{@tagName(op)}); - } - - const lhs_wasm_bits = toWasmBits(@intCast(lhs_ty.bitSize(zcu))) orelse { - return cg.fail("TODO: implement '{s}' for types larger than 128 bits", .{@tagName(op)}); - }; - const rhs_wasm_bits = toWasmBits(@intCast(rhs_ty.bitSize(zcu))).?; - const new_rhs = if (lhs_wasm_bits != rhs_wasm_bits and lhs_wasm_bits != 128) - try (try cg.intcast(rhs, rhs_ty, lhs_ty)).toLocal(cg, lhs_ty) - else - rhs; - break :result try cg.binOp(lhs, new_rhs, lhs_ty, op); - }, - else => try cg.binOp(lhs, rhs, lhs_ty, op), - }; - - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -/// Performs a binary operation on the given `WValue`'s -/// NOTE: THis leaves the value on top of the stack. -fn binOp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { - const pt = cg.pt; - const zcu = pt.zcu; - assert(!(lhs != .stack and rhs == .stack)); - - if (ty.isAnyFloat()) { - const float_op = FloatOp.fromOp(op); - return cg.floatOp(float_op, ty, &.{ lhs, rhs }); - } - - if (isByRef(ty, zcu, cg.target)) { - if (ty.zigTypeTag(zcu) == .int) { - return cg.binOpBigInt(lhs, rhs, ty, op); - } else { - return cg.fail("TODO: Implement binary operation for type: {f}", .{ty.fmt(pt)}); - } - } - - const opcode: std.wasm.Opcode = buildOpcode(.{ - .op = op, - .valtype1 = typeToValtype(ty, zcu, cg.target), - .signedness = if (ty.isSignedInt(zcu)) .signed else .unsigned, - }); - try cg.emitWValue(lhs); - try cg.emitWValue(rhs); - - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - - return .stack; -} - -fn binOpBigInt(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { - const zcu = cg.pt.zcu; - const int_info = ty.intInfo(zcu); - if (int_info.bits > 128) { - return cg.fail("TODO: Implement binary operation for big integers larger than 128 bits", .{}); - } - - switch (op) { - .mul => return cg.callIntrinsic(.__multi3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), - .div => switch (int_info.signedness) { - .signed => return cg.callIntrinsic(.__divti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), - .unsigned => return cg.callIntrinsic(.__udivti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), - }, - .rem => switch (int_info.signedness) { - .signed => return cg.callIntrinsic(.__modti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), - .unsigned => return cg.callIntrinsic(.__umodti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), - }, - .shr => switch (int_info.signedness) { - .signed => return cg.callIntrinsic(.__ashrti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }), - .unsigned => return cg.callIntrinsic(.__lshrti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }), - }, - .shl => return cg.callIntrinsic(.__ashlti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }), - .@"and", .@"or", .xor => { - const result = try cg.allocStack(ty); - try cg.emitWValue(result); - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); - const op_lsb = try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, op); - try cg.store(.stack, op_lsb, Type.u64, result.offset()); - - try cg.emitWValue(result); - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); - const op_msb = try cg.binOp(lhs_msb, rhs_msb, Type.u64, op); - try cg.store(.stack, op_msb, Type.u64, result.offset() + 8); - return result; - }, - .add, .sub => { - const result = try cg.allocStack(ty); - var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64); - defer lhs_lsb.free(cg); - var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64); - defer rhs_lsb.free(cg); - var op_lsb = try (try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, op)).toLocal(cg, Type.u64); - defer op_lsb.free(cg); - - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); - const op_msb = try cg.binOp(lhs_msb, rhs_msb, Type.u64, op); - - const lt = if (op == .add) blk: { - break :blk try cg.cmp(op_lsb, rhs_lsb, Type.u64, .lt); - } else if (op == .sub) blk: { - break :blk try cg.cmp(lhs_lsb, rhs_lsb, Type.u64, .lt); - } else unreachable; - const tmp = try cg.intcast(lt, Type.u32, Type.u64); - var tmp_op = try (try cg.binOp(op_msb, tmp, Type.u64, op)).toLocal(cg, Type.u64); - defer tmp_op.free(cg); - - try cg.store(result, op_lsb, Type.u64, 0); - try cg.store(result, tmp_op, Type.u64, 8); - return result; - }, - else => return cg.fail("TODO: Implement binary operation for big integers: '{s}'", .{@tagName(op)}), - } -} - -const FloatOp = enum { - add, - ceil, - cos, - div, - exp, - exp2, - fabs, - floor, - fma, - fmax, - fmin, - fmod, - log, - log10, - log2, - mul, - neg, - round, - sin, - sqrt, - sub, - tan, - trunc, - - pub fn fromOp(op: Op) FloatOp { - return switch (op) { - .add => .add, - .ceil => .ceil, - .div => .div, - .abs => .fabs, - .floor => .floor, - .max => .fmax, - .min => .fmin, - .mul => .mul, - .neg => .neg, - .nearest => .round, - .sqrt => .sqrt, - .sub => .sub, - .trunc => .trunc, - .rem => .fmod, - else => unreachable, - }; - } - - pub fn toOp(float_op: FloatOp) ?Op { - return switch (float_op) { - .add => .add, - .ceil => .ceil, - .div => .div, - .fabs => .abs, - .floor => .floor, - .fmax => .max, - .fmin => .min, - .mul => .mul, - .neg => .neg, - .round => .nearest, - .sqrt => .sqrt, - .sub => .sub, - .trunc => .trunc, - - .cos, - .exp, - .exp2, - .fma, - .fmod, - .log, - .log10, - .log2, - .sin, - .tan, - => null, - }; - } - - fn intrinsic(op: FloatOp, bits: u16) Mir.Intrinsic { - return switch (op) { - inline .add, .sub, .div, .mul => |ct_op| switch (bits) { - inline 16, 80, 128 => |ct_bits| @field( - Mir.Intrinsic, - "__" ++ @tagName(ct_op) ++ compilerRtFloatAbbrev(ct_bits) ++ "f3", - ), - else => unreachable, - }, - - inline .ceil, - .fabs, - .floor, - .fmax, - .fmin, - .round, - .sqrt, - .trunc, - => |ct_op| switch (bits) { - inline 16, 80, 128 => |ct_bits| @field( - Mir.Intrinsic, - libcFloatPrefix(ct_bits) ++ @tagName(ct_op) ++ libcFloatSuffix(ct_bits), - ), - else => unreachable, - }, - - inline .cos, - .exp, - .exp2, - .fma, - .fmod, - .log, - .log10, - .log2, - .sin, - .tan, - => |ct_op| switch (bits) { - inline 16, 32, 64, 80, 128 => |ct_bits| @field( - Mir.Intrinsic, - libcFloatPrefix(ct_bits) ++ @tagName(ct_op) ++ libcFloatSuffix(ct_bits), - ), - else => unreachable, - }, - - .neg => unreachable, - }; - } -}; - -fn airAbs(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - const ty = cg.typeOf(ty_op.operand); - const scalar_ty = ty.scalarType(zcu); - - switch (scalar_ty.zigTypeTag(zcu)) { - .int => if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)}); - } else { - const int_bits = ty.intInfo(zcu).bits; - const wasm_bits = toWasmBits(int_bits) orelse { - return cg.fail("TODO: airAbs for signed integers larger than '{d}' bits", .{int_bits}); - }; - - switch (wasm_bits) { - 32 => { - try cg.emitWValue(operand); - - try cg.addImm32(31); - try cg.addTag(.i32_shr_s); - - var tmp = try cg.allocLocal(ty); - defer tmp.free(cg); - try cg.addLocal(.local_tee, tmp.local.value); - - try cg.emitWValue(operand); - try cg.addTag(.i32_xor); - try cg.emitWValue(tmp); - try cg.addTag(.i32_sub); - return cg.finishAir(inst, .stack, &.{ty_op.operand}); - }, - 64 => { - try cg.emitWValue(operand); - - try cg.addImm64(63); - try cg.addTag(.i64_shr_s); - - var tmp = try cg.allocLocal(ty); - defer tmp.free(cg); - try cg.addLocal(.local_tee, tmp.local.value); - - try cg.emitWValue(operand); - try cg.addTag(.i64_xor); - try cg.emitWValue(tmp); - try cg.addTag(.i64_sub); - return cg.finishAir(inst, .stack, &.{ty_op.operand}); - }, - 128 => { - const mask = try cg.allocStack(Type.u128); - try cg.emitWValue(mask); - try cg.emitWValue(mask); - - _ = try cg.load(operand, Type.u64, 8); - try cg.addImm64(63); - try cg.addTag(.i64_shr_s); - - var tmp = try cg.allocLocal(Type.u64); - defer tmp.free(cg); - try cg.addLocal(.local_tee, tmp.local.value); - try cg.store(.stack, .stack, Type.u64, mask.offset() + 0); - try cg.emitWValue(tmp); - try cg.store(.stack, .stack, Type.u64, mask.offset() + 8); - - const a = try cg.binOpBigInt(operand, mask, Type.u128, .xor); - const b = try cg.binOpBigInt(a, mask, Type.u128, .sub); - - return cg.finishAir(inst, b, &.{ty_op.operand}); - }, - else => unreachable, - } - }, - .float => { - const result = try cg.floatOp(.fabs, ty, &.{operand}); - return cg.finishAir(inst, result, &.{ty_op.operand}); - }, - else => unreachable, - } -} - -fn airUnaryFloatOp(cg: *CodeGen, inst: Air.Inst.Index, op: FloatOp) InnerError!void { - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - const ty = cg.typeOf(un_op); - - const result = try cg.floatOp(op, ty, &.{operand}); - return cg.finishAir(inst, result, &.{un_op}); -} - -fn floatOp(cg: *CodeGen, float_op: FloatOp, ty: Type, args: []const WValue) InnerError!WValue { - const zcu = cg.pt.zcu; - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: Implement floatOps for vectors", .{}); - } - - const float_bits = ty.floatBits(cg.target); - - if (float_op == .neg) { - return cg.floatNeg(ty, args[0]); - } - - if (float_bits == 32 or float_bits == 64) { - if (float_op.toOp()) |op| { - for (args) |operand| { - try cg.emitWValue(operand); - } - const opcode = buildOpcode(.{ .op = op, .valtype1 = typeToValtype(ty, zcu, cg.target) }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - return .stack; - } - } - - const intrinsic = float_op.intrinsic(float_bits); - - // fma requires three operands - var param_types_buffer: [3]InternPool.Index = .{ ty.ip_index, ty.ip_index, ty.ip_index }; - const param_types = param_types_buffer[0..args.len]; - return cg.callIntrinsic(intrinsic, param_types, ty, args); -} - -/// NOTE: The result value remains on top of the stack. -fn floatNeg(cg: *CodeGen, ty: Type, arg: WValue) InnerError!WValue { - const float_bits = ty.floatBits(cg.target); - switch (float_bits) { - 16 => { - try cg.emitWValue(arg); - try cg.addImm32(0x8000); - try cg.addTag(.i32_xor); - return .stack; - }, - 32, 64 => { - try cg.emitWValue(arg); - const val_type: std.wasm.Valtype = if (float_bits == 32) .f32 else .f64; - const opcode = buildOpcode(.{ .op = .neg, .valtype1 = val_type }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - return .stack; - }, - 80, 128 => { - const result = try cg.allocStack(ty); - try cg.emitWValue(result); - try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 }); - try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 }); - - try cg.emitWValue(result); - try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 }); - - if (float_bits == 80) { - try cg.addImm64(0x8000); - try cg.addTag(.i64_xor); - try cg.addMemArg(.i64_store16, .{ .offset = 8 + result.offset(), .alignment = 2 }); - } else { - try cg.addImm64(0x8000000000000000); - try cg.addTag(.i64_xor); - try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 }); - } - return result; - }, - else => unreachable, - } -} - -fn airWrapBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const lhs_ty = cg.typeOf(bin_op.lhs); - const rhs_ty = cg.typeOf(bin_op.rhs); - - if (lhs_ty.isVector(zcu)) { - if ((op == .shr or op == .shl) and !rhs_ty.isVector(zcu)) { - return cg.fail("TODO: implement wrapping vector '{s}' with scalar rhs", .{@tagName(op)}); - } else { - return cg.fail("TODO: implement wrapping '{s}' for vectors", .{@tagName(op)}); - } - } - - // For certain operations, such as shifting, the types are different. - // When converting this to a WebAssembly type, they *must* match to perform - // an operation. For this reason we verify if the WebAssembly type is different, in which - // case we first coerce the operands to the same type before performing the operation. - // For big integers we can ignore this as we will call into compiler-rt which handles this. - const result = switch (op) { - .shr, .shl => result: { - const lhs_wasm_bits = toWasmBits(@intCast(lhs_ty.bitSize(zcu))) orelse { - return cg.fail("TODO: implement '{s}' for types larger than 128 bits", .{@tagName(op)}); - }; - const rhs_wasm_bits = toWasmBits(@intCast(rhs_ty.bitSize(zcu))).?; - const new_rhs = if (lhs_wasm_bits != rhs_wasm_bits and lhs_wasm_bits != 128) - try (try cg.intcast(rhs, rhs_ty, lhs_ty)).toLocal(cg, lhs_ty) - else - rhs; - break :result try cg.wrapBinOp(lhs, new_rhs, lhs_ty, op); - }, - else => try cg.wrapBinOp(lhs, rhs, lhs_ty, op), - }; - - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -/// Performs a wrapping binary operation. -/// Asserts rhs is not a stack value when lhs also isn't. -/// NOTE: Leaves the result on the stack when its Type is <= 64 bits -fn wrapBinOp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { - const bin_local = try cg.binOp(lhs, rhs, ty, op); - return cg.wrapOperand(bin_local, ty); -} - -/// Wraps an operand based on a given type's bitsize. -/// Asserts `Type` is <= 128 bits. -/// NOTE: When the Type is <= 64 bits, leaves the value on top of the stack, if wrapping was needed. -fn wrapOperand(cg: *CodeGen, operand: WValue, ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - assert(ty.abiSize(zcu) <= 16); - const int_bits: u16 = @intCast(ty.bitSize(zcu)); // TODO use ty.intInfo(zcu).bits - const wasm_bits = toWasmBits(int_bits) orelse { - return cg.fail("TODO: Implement wrapOperand for bitsize '{d}'", .{int_bits}); - }; - - if (wasm_bits == int_bits) return operand; - - switch (wasm_bits) { - 32 => { - try cg.emitWValue(operand); - if (ty.isSignedInt(zcu)) { - try cg.addImm32(32 - int_bits); - try cg.addTag(.i32_shl); - try cg.addImm32(32 - int_bits); - try cg.addTag(.i32_shr_s); - } else { - try cg.addImm32(~@as(u32, 0) >> @intCast(32 - int_bits)); - try cg.addTag(.i32_and); - } - return .stack; - }, - 64 => { - try cg.emitWValue(operand); - if (ty.isSignedInt(zcu)) { - try cg.addImm64(64 - int_bits); - try cg.addTag(.i64_shl); - try cg.addImm64(64 - int_bits); - try cg.addTag(.i64_shr_s); - } else { - try cg.addImm64(~@as(u64, 0) >> @intCast(64 - int_bits)); - try cg.addTag(.i64_and); - } - return .stack; - }, - 128 => { - assert(operand != .stack); - const result = try cg.allocStack(ty); - - try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, 0); - try cg.store(.stack, .stack, Type.u64, result.offset()); - - try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, 8); - if (ty.isSignedInt(zcu)) { - try cg.addImm64(128 - int_bits); - try cg.addTag(.i64_shl); - try cg.addImm64(128 - int_bits); - try cg.addTag(.i64_shr_s); - } else { - try cg.addImm64(~@as(u64, 0) >> @intCast(128 - int_bits)); - try cg.addTag(.i64_and); - } - try cg.store(.stack, .stack, Type.u64, result.offset() + 8); - - return result; - }, - else => unreachable, - } -} - -fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue { - const pt = cg.pt; - const zcu = pt.zcu; - const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr; - const offset: u64 = prev_offset + ptr.byte_offset; - return switch (ptr.base_addr) { - .nav => |nav| return .{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } }, - .uav => |uav| return .{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } }, - .int => return cg.lowerConstant(try pt.intValue(Type.usize, offset), Type.usize), - .eu_payload => |eu_ptr| try cg.lowerPtr( - eu_ptr, - offset + codegen.errUnionPayloadOffset( - Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu), - zcu, - ), - ), - .opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset), - .field => |field| { - const base_ptr = Value.fromInterned(field.base); - const base_ty = base_ptr.typeOf(zcu).childType(zcu); - const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) { - .pointer => off: { - assert(base_ty.isSlice(zcu)); - break :off switch (field.index) { - Value.slice_ptr_index => 0, - Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8), - else => unreachable, - }; - }, - .@"struct" => switch (base_ty.containerLayout(zcu)) { - .auto => base_ty.structFieldOffset(@intCast(field.index), zcu), - .@"extern", .@"packed" => unreachable, - }, - .@"union" => switch (base_ty.containerLayout(zcu)) { - .auto => base_ty.structFieldOffset(@intCast(field.index), zcu), - .@"extern", .@"packed" => unreachable, - }, - else => unreachable, - }; - return cg.lowerPtr(field.base, offset + field_off); - }, - .arr_elem, .comptime_field, .comptime_alloc => unreachable, - }; -} - -/// Asserts that `isByRef` returns `false` for `ty`. -fn lowerConstant(cg: *CodeGen, val: Value, ty: Type) InnerError!WValue { - const pt = cg.pt; - const zcu = pt.zcu; - assert(!isByRef(ty, zcu, cg.target)); - const ip = &zcu.intern_pool; - if (val.isUndef(zcu)) return cg.emitUndefined(ty); - - switch (ip.indexToKey(val.ip_index)) { - .int_type, - .ptr_type, - .array_type, - .vector_type, - .opt_type, - .anyframe_type, - .error_union_type, - .simple_type, - .struct_type, - .tuple_type, - .union_type, - .opaque_type, - .enum_type, - .func_type, - .error_set_type, - .inferred_error_set_type, - => unreachable, // types, not values - - .undef => unreachable, // handled above - .simple_value => |simple_value| switch (simple_value) { - .undefined, - .void, - .null, - .empty_tuple, - .@"unreachable", - => unreachable, // non-runtime values - .false, .true => return .{ .imm32 = switch (simple_value) { - .false => 0, - .true => 1, - else => unreachable, - } }, - }, - .variable, - .@"extern", - .func, - .enum_literal, - .empty_enum_value, - => unreachable, // non-runtime values - .int => { - const int_info = ty.intInfo(zcu); - switch (int_info.signedness) { - .signed => switch (int_info.bits) { - 0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) }, - 33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) }, - else => unreachable, - }, - .unsigned => switch (int_info.bits) { - 0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) }, - 33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) }, - else => unreachable, - }, - } - }, - .err => |err| { - const int = try pt.getErrorValue(err.name); - return .{ .imm32 = int }; - }, - .error_union => |error_union| { - const err_int_ty = try pt.errorIntType(); - const err_ty, const err_val = switch (error_union.val) { - .err_name => |err_name| .{ - ty.errorUnionSet(zcu), - Value.fromInterned(try pt.intern(.{ .err = .{ - .ty = ty.errorUnionSet(zcu).toIntern(), - .name = err_name, - } })), - }, - .payload => .{ - err_int_ty, - try pt.intValue(err_int_ty, 0), - }, - }; - const payload_type = ty.errorUnionPayload(zcu); - if (!payload_type.hasRuntimeBitsIgnoreComptime(zcu)) { - // We use the error type directly as the type. - return cg.lowerConstant(err_val, err_ty); - } - - return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{}); - }, - .enum_tag => |enum_tag| { - const int_tag_ty = ip.typeOf(enum_tag.int); - return cg.lowerConstant(Value.fromInterned(enum_tag.int), Type.fromInterned(int_tag_ty)); - }, - .float => |float| switch (float.storage) { - .f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) }, - .f32 => |f32_val| return .{ .float32 = f32_val }, - .f64 => |f64_val| return .{ .float64 = f64_val }, - else => unreachable, - }, - .slice => unreachable, // isByRef == true - .ptr => return cg.lowerPtr(val.toIntern(), 0), - .opt => if (ty.optionalReprIsPayload(zcu)) { - const pl_ty = ty.optionalChild(zcu); - if (val.optionalValue(zcu)) |payload| { - return cg.lowerConstant(payload, pl_ty); - } else { - return .{ .imm32 = 0 }; - } - } else { - return .{ .imm32 = @intFromBool(!val.isNull(zcu)) }; - }, - .aggregate => switch (ip.indexToKey(ty.ip_index)) { - .array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}), - .vector_type => { - assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct); - var buf: [16]u8 = undefined; - val.writeToMemory(pt, &buf) catch unreachable; - return cg.storeSimdImmd(buf); - }, - .struct_type => { - const struct_type = ip.loadStructType(ty.toIntern()); - // non-packed structs are not handled in this function because they - // are by-ref types. - assert(struct_type.layout == .@"packed"); - var buf: [8]u8 = .{0} ** 8; // zero the buffer so we do not read 0xaa as integer - val.writeToPackedMemory(ty, pt, &buf, 0) catch unreachable; - const backing_int_ty = Type.fromInterned(struct_type.backingIntTypeUnordered(ip)); - const int_val = try pt.intValue( - backing_int_ty, - mem.readInt(u64, &buf, .little), - ); - return cg.lowerConstant(int_val, backing_int_ty); - }, - else => unreachable, - }, - .un => { - const int_type = try pt.intType(.unsigned, @intCast(ty.bitSize(zcu))); - - var buf: [8]u8 = .{0} ** 8; // zero the buffer so we do not read 0xaa as integer - val.writeToPackedMemory(ty, pt, &buf, 0) catch unreachable; - const int_val = try pt.intValue( - int_type, - mem.readInt(u64, &buf, .little), - ); - return cg.lowerConstant(int_val, int_type); - }, - .memoized_call => unreachable, - } -} - -/// Stores the value as a 128bit-immediate value by storing it inside -/// the list and returning the index into this list as `WValue`. -fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue { - const index = @as(u32, @intCast(cg.simd_immediates.items.len)); - try cg.simd_immediates.append(cg.gpa, value); - return .{ .imm128 = index }; -} - -fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - switch (ty.zigTypeTag(zcu)) { - .bool, .error_set => return .{ .imm32 = 0xaaaaaaaa }, - .int, .@"enum" => switch (ty.intInfo(zcu).bits) { - 0...32 => return .{ .imm32 = 0xaaaaaaaa }, - 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, - else => unreachable, - }, - .float => switch (ty.floatBits(cg.target)) { - 16 => return .{ .imm32 = 0xaaaaaaaa }, - 32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) }, - 64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) }, - else => unreachable, - }, - .pointer => switch (cg.ptr_size) { - .wasm32 => return .{ .imm32 = 0xaaaaaaaa }, - .wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, - }, - .optional => { - const pl_ty = ty.optionalChild(zcu); - if (ty.optionalReprIsPayload(zcu)) { - return cg.emitUndefined(pl_ty); - } - return .{ .imm32 = 0xaaaaaaaa }; - }, - .error_union => { - return .{ .imm32 = 0xaaaaaaaa }; - }, - .@"struct" => { - const packed_struct = zcu.typeToPackedStruct(ty).?; - return cg.emitUndefined(Type.fromInterned(packed_struct.backingIntTypeUnordered(ip))); - }, - .@"union" => switch (ty.containerLayout(zcu)) { - .@"packed" => switch (ty.bitSize(zcu)) { - 0...32 => return .{ .imm32 = 0xaaaaaaaa }, - 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, - else => unreachable, - }, - else => unreachable, - }, - else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}), - } -} - -fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.Block, ty_pl.payload); - try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); -} - -fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - // if wasm_block_ty is non-empty, we create a register to store the temporary value - const block_result: WValue = if (block_ty.hasRuntimeBitsIgnoreComptime(zcu)) - try cg.allocLocal(block_ty) - else - .none; - - try cg.startBlock(.block, .empty); - // Here we set the current block idx, so breaks know the depth to jump - // to when breaking out. - try cg.blocks.putNoClobber(cg.gpa, inst, .{ - .label = cg.block_depth, - .value = block_result, - }); - - try cg.genBody(body); - try cg.endBlock(); - - const liveness = cg.liveness.getBlock(inst); - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths.len); - - return cg.finishAir(inst, block_result, &.{}); -} - -/// appends a new wasm block to the code section and increases the `block_depth` by 1 -fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void { - cg.block_depth += 1; - try cg.addInst(.{ - .tag = Mir.Inst.Tag.fromOpcode(block_tag), - .data = .{ .block_type = block_type }, - }); -} - -/// Ends the current wasm block and decreases the `block_depth` by 1 -fn endBlock(cg: *CodeGen) !void { - try cg.addTag(.end); - cg.block_depth -= 1; -} - -fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const loop = cg.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]); - - // result type of loop is always 'noreturn', meaning we can always - // emit the wasm type 'block_empty'. - try cg.startBlock(.loop, .empty); - - try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth); - defer assert(cg.loops.remove(inst)); - - try cg.genBody(body); - try cg.endBlock(); - - return cg.finishAir(inst, .none, &.{}); -} - -fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition = try cg.resolveInst(pl_op.operand); - const extra = cg.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); - const liveness_condbr = cg.liveness.getCondBr(inst); - - // result type is always noreturn, so use `block_empty` as type. - try cg.startBlock(.block, .empty); - // emit the conditional value - try cg.emitWValue(condition); - - // we inserted the block in front of the condition - // so now check if condition matches. If not, break outside this block - // and continue with the then codepath - try cg.addLabel(.br_if, 0); - - try cg.branches.ensureUnusedCapacity(cg.gpa, 2); - { - cg.branches.appendAssumeCapacity(.{}); - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.else_deaths.len))); - defer { - var else_stack = cg.branches.pop().?; - else_stack.deinit(cg.gpa); - } - try cg.genBody(else_body); - try cg.endBlock(); - } - - // Outer block that matches the condition - { - cg.branches.appendAssumeCapacity(.{}); - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.then_deaths.len))); - defer { - var then_stack = cg.branches.pop().?; - then_stack.deinit(cg.gpa); - } - try cg.genBody(then_body); - } - - return cg.finishAir(inst, .none, &.{}); -} - -fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const operand_ty = cg.typeOf(bin_op.lhs); - const result = try cg.cmp(lhs, rhs, operand_ty, op); - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -/// Compares two operands. -/// Asserts rhs is not a stack value when the lhs isn't a stack value either -/// NOTE: This leaves the result on top of the stack, rather than a new local. -fn cmp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOperator) InnerError!WValue { - assert(!(lhs != .stack and rhs == .stack)); - const zcu = cg.pt.zcu; - if (ty.zigTypeTag(zcu) == .optional and !ty.optionalReprIsPayload(zcu)) { - const payload_ty = ty.optionalChild(zcu); - if (payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - // When we hit this case, we must check the value of optionals - // that are not pointers. This means first checking against non-null for - // both lhs and rhs, as well as checking the payload are matching of lhs and rhs - return cg.cmpOptionals(lhs, rhs, ty, op); - } - } else if (ty.isAnyFloat()) { - return cg.cmpFloat(ty, lhs, rhs, op); - } else if (isByRef(ty, zcu, cg.target)) { - return cg.cmpBigInt(lhs, rhs, ty, op); - } - - const signedness: std.builtin.Signedness = blk: { - // by default we tell the operand type is unsigned (i.e. bools and enum values) - if (ty.zigTypeTag(zcu) != .int) break :blk .unsigned; - - // incase of an actual integer, we emit the correct signedness - break :blk ty.intInfo(zcu).signedness; - }; - - // ensure that when we compare pointers, we emit - // the true pointer of a stack value, rather than the stack pointer. - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); - - const opcode: std.wasm.Opcode = buildOpcode(.{ - .valtype1 = typeToValtype(ty, zcu, cg.target), - .op = switch (op) { - .lt => .lt, - .lte => .le, - .eq => .eq, - .neq => .ne, - .gte => .ge, - .gt => .gt, - }, - .signedness = signedness, - }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - - return .stack; -} - -/// Compares two floats. -/// NOTE: Leaves the result of the comparison on top of the stack. -fn cmpFloat(cg: *CodeGen, ty: Type, lhs: WValue, rhs: WValue, cmp_op: std.math.CompareOperator) InnerError!WValue { - const float_bits = ty.floatBits(cg.target); - - const op: Op = switch (cmp_op) { - .lt => .lt, - .lte => .le, - .eq => .eq, - .neq => .ne, - .gte => .ge, - .gt => .gt, - }; - - switch (float_bits) { - 16 => { - _ = try cg.fpext(lhs, Type.f16, Type.f32); - _ = try cg.fpext(rhs, Type.f16, Type.f32); - const opcode = buildOpcode(.{ .op = op, .valtype1 = .f32 }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - return .stack; - }, - 32, 64 => { - try cg.emitWValue(lhs); - try cg.emitWValue(rhs); - const val_type: std.wasm.Valtype = if (float_bits == 32) .f32 else .f64; - const opcode = buildOpcode(.{ .op = op, .valtype1 = val_type }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - return .stack; - }, - 80, 128 => { - const intrinsic = floatCmpIntrinsic(cmp_op, float_bits); - const result = try cg.callIntrinsic(intrinsic, &.{ ty.ip_index, ty.ip_index }, Type.bool, &.{ lhs, rhs }); - return cg.cmp(result, .{ .imm32 = 0 }, Type.i32, cmp_op); - }, - else => unreachable, - } -} - -fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - _ = inst; - return cg.fail("TODO implement airCmpVector for wasm", .{}); -} - -fn airCmpLtErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - - try cg.emitWValue(operand); - const pt = cg.pt; - const err_int_ty = try pt.errorIntType(); - try cg.addTag(.errors_len); - const result = try cg.cmp(.stack, .stack, err_int_ty, .lt); - - return cg.finishAir(inst, result, &.{un_op}); -} - -fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br; - const block = cg.blocks.get(br.block_inst).?; - - // if operand has codegen bits we should break with a value - if (block.value != .none) { - const operand = try cg.resolveInst(br.operand); - try cg.lowerToStack(operand); - try cg.addLocal(.local_set, block.value.local.value); - } - - // We map every block to its block index. - // We then determine how far we have to jump to it by subtracting it from current block depth - const idx: u32 = cg.block_depth - block.label; - try cg.addLabel(.br, idx); - - return cg.finishAir(inst, .none, &.{br.operand}); -} - -fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const repeat = cg.air.instructions.items(.data)[@intFromEnum(inst)].repeat; - const loop_label = cg.loops.get(repeat.loop_inst).?; - - const idx: u32 = cg.block_depth - loop_label; - try cg.addLabel(.br, idx); - - return cg.finishAir(inst, .none, &.{}); -} - -fn airNot(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const operand_ty = cg.typeOf(ty_op.operand); - const pt = cg.pt; - const zcu = pt.zcu; - - const result = result: { - if (operand_ty.zigTypeTag(zcu) == .bool) { - try cg.emitWValue(operand); - try cg.addTag(.i32_eqz); - const not_tmp = try cg.allocLocal(operand_ty); - try cg.addLocal(.local_set, not_tmp.local.value); - break :result not_tmp; - } else { - const int_info = operand_ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: Implement binary NOT for {f}", .{operand_ty.fmt(pt)}); - }; - - switch (wasm_bits) { - 32 => { - try cg.emitWValue(operand); - try cg.addImm32(switch (int_info.signedness) { - .unsigned => ~@as(u32, 0) >> @intCast(32 - int_info.bits), - .signed => ~@as(u32, 0), - }); - try cg.addTag(.i32_xor); - break :result .stack; - }, - 64 => { - try cg.emitWValue(operand); - try cg.addImm64(switch (int_info.signedness) { - .unsigned => ~@as(u64, 0) >> @intCast(64 - int_info.bits), - .signed => ~@as(u64, 0), - }); - try cg.addTag(.i64_xor); - break :result .stack; - }, - 128 => { - const ptr = try cg.allocStack(operand_ty); - - try cg.emitWValue(ptr); - _ = try cg.load(operand, Type.u64, 0); - try cg.addImm64(~@as(u64, 0)); - try cg.addTag(.i64_xor); - try cg.store(.stack, .stack, Type.u64, ptr.offset()); - - try cg.emitWValue(ptr); - _ = try cg.load(operand, Type.u64, 8); - try cg.addImm64(switch (int_info.signedness) { - .unsigned => ~@as(u64, 0) >> @intCast(128 - int_info.bits), - .signed => ~@as(u64, 0), - }); - try cg.addTag(.i64_xor); - try cg.store(.stack, .stack, Type.u64, ptr.offset() + 8); - - break :result ptr; - }, - else => unreachable, - } - } - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - try cg.addTag(.@"unreachable"); - return cg.finishAir(inst, .none, &.{}); -} - -fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - // unsupported by wasm itfunc. Can be implemented once we support DWARF - // for wasm - try cg.addTag(.@"unreachable"); - return cg.finishAir(inst, .none, &.{}); -} - -fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - try cg.addTag(.@"unreachable"); - return cg.finishAir(inst, .none, &.{}); -} - -fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - const wanted_ty = cg.typeOfIndex(inst); - const given_ty = cg.typeOf(ty_op.operand); - - const bit_size = given_ty.bitSize(zcu); - const needs_wrapping = (given_ty.isSignedInt(zcu) != wanted_ty.isSignedInt(zcu)) and - bit_size != 32 and bit_size != 64 and bit_size != 128; - - const result = result: { - if (given_ty.isAnyFloat() or wanted_ty.isAnyFloat()) { - break :result try cg.bitcast(wanted_ty, given_ty, operand); - } - - if (isByRef(given_ty, zcu, cg.target) and !isByRef(wanted_ty, zcu, cg.target)) { - const loaded_memory = try cg.load(operand, wanted_ty, 0); - if (needs_wrapping) { - break :result try cg.wrapOperand(loaded_memory, wanted_ty); - } else { - break :result loaded_memory; - } - } - if (!isByRef(given_ty, zcu, cg.target) and isByRef(wanted_ty, zcu, cg.target)) { - const stack_memory = try cg.allocStack(wanted_ty); - try cg.store(stack_memory, operand, given_ty, 0); - if (needs_wrapping) { - break :result try cg.wrapOperand(stack_memory, wanted_ty); - } else { - break :result stack_memory; - } - } - - if (needs_wrapping) { - break :result try cg.wrapOperand(operand, wanted_ty); - } - - break :result switch (operand) { - // for stack offset, return a pointer to this offset. - .stack_offset => try cg.buildPointerOffset(operand, 0, .new), - else => cg.reuseOperand(ty_op.operand, operand), - }; - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn bitcast(cg: *CodeGen, wanted_ty: Type, given_ty: Type, operand: WValue) InnerError!WValue { - const zcu = cg.pt.zcu; - // if we bitcast a float to or from an integer we must use the 'reinterpret' instruction - if (!(wanted_ty.isAnyFloat() or given_ty.isAnyFloat())) return operand; - if (wanted_ty.ip_index == .f16_type or given_ty.ip_index == .f16_type) return operand; - if (wanted_ty.bitSize(zcu) > 64) return operand; - assert((wanted_ty.isInt(zcu) and given_ty.isAnyFloat()) or (wanted_ty.isAnyFloat() and given_ty.isInt(zcu))); - - const opcode = buildOpcode(.{ - .op = .reinterpret, - .valtype1 = typeToValtype(wanted_ty, zcu, cg.target), - .valtype2 = typeToValtype(given_ty, zcu, cg.target), - }); - try cg.emitWValue(operand); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - return .stack; -} - -fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.StructField, ty_pl.payload); - - const struct_ptr = try cg.resolveInst(extra.data.struct_operand); - const struct_ptr_ty = cg.typeOf(extra.data.struct_operand); - const struct_ty = struct_ptr_ty.childType(zcu); - const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index); - return cg.finishAir(inst, result, &.{extra.data.struct_operand}); -} - -fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const struct_ptr = try cg.resolveInst(ty_op.operand); - const struct_ptr_ty = cg.typeOf(ty_op.operand); - const struct_ty = struct_ptr_ty.childType(zcu); - - const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn structFieldPtr( - cg: *CodeGen, - inst: Air.Inst.Index, - ref: Air.Inst.Ref, - struct_ptr: WValue, - struct_ptr_ty: Type, - struct_ty: Type, - index: u32, -) InnerError!WValue { - const pt = cg.pt; - const zcu = pt.zcu; - const result_ty = cg.typeOfIndex(inst); - const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu); - - const offset = switch (struct_ty.containerLayout(zcu)) { - .@"packed" => switch (struct_ty.zigTypeTag(zcu)) { - .@"struct" => offset: { - if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) { - break :offset @as(u32, 0); - } - const struct_type = zcu.typeToStruct(struct_ty).?; - break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8); - }, - .@"union" => 0, - else => unreachable, - }, - else => struct_ty.structFieldOffset(index, zcu), - }; - // save a load and store when we can simply reuse the operand - if (offset == 0) { - return cg.reuseOperand(ref, struct_ptr); - } - switch (struct_ptr) { - .stack_offset => |stack_offset| { - return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } }; - }, - else => return cg.buildPointerOffset(struct_ptr, offset, .new), - } -} - -fn airStructFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; - - const struct_ty = cg.typeOf(struct_field.struct_operand); - const operand = try cg.resolveInst(struct_field.struct_operand); - const field_index = struct_field.field_index; - const field_ty = struct_ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand}); - - const result: WValue = switch (struct_ty.containerLayout(zcu)) { - .@"packed" => switch (struct_ty.zigTypeTag(zcu)) { - .@"struct" => result: { - const packed_struct = zcu.typeToPackedStruct(struct_ty).?; - const offset = zcu.structPackedFieldBitOffset(packed_struct, field_index); - const backing_ty = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); - const host_bits = backing_ty.intInfo(zcu).bits; - - const const_wvalue: WValue = if (33 <= host_bits and host_bits <= 64) - .{ .imm64 = offset } - else - .{ .imm32 = offset }; - - // for first field we don't require any shifting - const shifted_value = if (offset == 0) - operand - else - try cg.binOp(operand, const_wvalue, backing_ty, .shr); - - if (field_ty.zigTypeTag(zcu) == .float) { - const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); - const truncated = try cg.trunc(shifted_value, int_type, backing_ty); - break :result try cg.bitcast(field_ty, int_type, truncated); - } else if (field_ty.isPtrAtRuntime(zcu) and packed_struct.field_types.len == 1) { - // In this case we do not have to perform any transformations, - // we can simply reuse the operand. - break :result cg.reuseOperand(struct_field.struct_operand, operand); - } else if (field_ty.isPtrAtRuntime(zcu)) { - const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); - break :result try cg.trunc(shifted_value, int_type, backing_ty); - } - break :result try cg.trunc(shifted_value, field_ty, backing_ty); - }, - .@"union" => result: { - if (isByRef(struct_ty, zcu, cg.target)) { - if (!isByRef(field_ty, zcu, cg.target)) { - break :result try cg.load(operand, field_ty, 0); - } else { - const new_stack_val = try cg.allocStack(field_ty); - try cg.store(new_stack_val, operand, field_ty, 0); - break :result new_stack_val; - } - } - - const union_int_type = try pt.intType(.unsigned, @as(u16, @intCast(struct_ty.bitSize(zcu)))); - if (field_ty.zigTypeTag(zcu) == .float) { - const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); - const truncated = try cg.trunc(operand, int_type, union_int_type); - break :result try cg.bitcast(field_ty, int_type, truncated); - } else if (field_ty.isPtrAtRuntime(zcu)) { - const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); - break :result try cg.trunc(operand, int_type, union_int_type); - } - break :result try cg.trunc(operand, field_ty, union_int_type); - }, - else => unreachable, - }, - else => result: { - const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse { - return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)}); - }; - if (isByRef(field_ty, zcu, cg.target)) { - switch (operand) { - .stack_offset => |stack_offset| { - break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } }; - }, - else => break :result try cg.buildPointerOffset(operand, offset, .new), - } - } - break :result try cg.load(operand, field_ty, offset); - }, - }; - - return cg.finishAir(inst, result, &.{struct_field.struct_operand}); -} - -fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - - const switch_br = cg.air.unwrapSwitch(inst); - const target_ty = cg.typeOf(switch_br.operand); - - assert(target_ty.hasRuntimeBitsIgnoreComptime(zcu)); - - // swap target value with placeholder local, for dispatching - const target = if (is_dispatch_loop) target: { - const initial_target = try cg.resolveInst(switch_br.operand); - const target: WValue = try cg.allocLocal(target_ty); - try cg.lowerToStack(initial_target); - try cg.addLocal(.local_set, target.local.value); - - try cg.startBlock(.loop, .empty); // dispatch loop start - try cg.blocks.putNoClobber(cg.gpa, inst, .{ - .label = cg.block_depth, - .value = target, - }); - - break :target target; - } else try cg.resolveInst(switch_br.operand); - - const liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1); - defer cg.gpa.free(liveness.deaths); - - const has_else_body = switch_br.else_body_len != 0; - const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions - try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body)); - - if (switch_br.cases_len == 0) { - assert(has_else_body); - - var it = switch_br.iterateCases(); - const else_body = it.elseBody(); - - cg.branches.appendAssumeCapacity(.{}); - const else_deaths = liveness.deaths.len - 1; - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len); - defer { - var else_branch = cg.branches.pop().?; - else_branch.deinit(cg.gpa); - } - try cg.genBody(else_body); - - if (is_dispatch_loop) { - try cg.endBlock(); // dispatch loop end - } - return cg.finishAir(inst, .none, &.{}); - } - - var min: ?Value = null; - var max: ?Value = null; - var branching_size: u32 = 0; // single item +1, range +2 - - { - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val; - if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val; - branching_size += 1; - } - for (case.ranges) |range| { - const low = Value.fromInterned(range[0].toInterned().?); - if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low; - const high = Value.fromInterned(range[1].toInterned().?); - if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high; - branching_size += 2; - } - } - } - - var min_space: Value.BigIntSpace = undefined; - const min_bigint = min.?.toBigInt(&min_space, zcu); - var max_space: Value.BigIntSpace = undefined; - const max_bigint = max.?.toBigInt(&max_space, zcu); - const limbs = try cg.gpa.alloc( - std.math.big.Limb, - @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1, - ); - defer cg.gpa.free(limbs); - - const width_maybe: ?u32 = width: { - var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - width_bigint.sub(max_bigint, min_bigint); - width_bigint.addScalar(width_bigint.toConst(), 1); - break :width width_bigint.toConst().toInt(u32) catch null; - }; - - try cg.startBlock(.block, .empty); // whole switch block start - - for (0..branch_count) |_| { - try cg.startBlock(.block, .empty); - } - - // Heuristic on deciding when to use .br_table instead of .br_if jump table - // 1. Differences between lowest and highest values should fit into u32 - // 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions - // 3. Do not use .br_table for tiny switches - const use_br_table = cond: { - const width = width_maybe orelse break :cond false; - if (width > 2 * branching_size) break :cond false; - if (width < 2 or branch_count < 2) break :cond false; - break :cond true; - }; - - if (use_br_table) { - const width = width_maybe.?; - - const br_value_original = try cg.binOp(target, try cg.resolveValue(min.?), target_ty, .sub); - _ = try cg.intcast(br_value_original, target_ty, Type.u32); - - const jump_table: Mir.JumpTable = .{ .length = width + 1 }; - const table_extra_index = try cg.addExtra(jump_table); - try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } }); - - const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1); - @memset(branch_list, branch_count - 1); - - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - var val_space: Value.BigIntSpace = undefined; - const val_bigint = val.toBigInt(&val_space, zcu); - var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - index_bigint.sub(val_bigint, min_bigint); - branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx; - } - for (case.ranges) |range| { - var low_space: Value.BigIntSpace = undefined; - const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu); - var high_space: Value.BigIntSpace = undefined; - const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu); - var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - index_bigint.sub(low_bigint, min_bigint); - const start = index_bigint.toConst().toInt(u32) catch unreachable; - index_bigint.sub(high_bigint, min_bigint); - const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1; - @memset(branch_list[start..end], case.idx); - } - } - } else { - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - for (case.items) |ref| { - const val = try cg.resolveInst(ref); - _ = try cg.cmp(target, val, target_ty, .eq); - try cg.addLabel(.br_if, case.idx); // item match found - } - for (case.ranges) |range| { - const low = try cg.resolveInst(range[0]); - const high = try cg.resolveInst(range[1]); - - const gte = try cg.cmp(target, low, target_ty, .gte); - const lte = try cg.cmp(target, high, target_ty, .lte); - _ = try cg.binOp(gte, lte, Type.bool, .@"and"); - try cg.addLabel(.br_if, case.idx); // range match found - } - } - try cg.addLabel(.br, branch_count - 1); - } - - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - try cg.endBlock(); - - cg.branches.appendAssumeCapacity(.{}); - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[case.idx].len); - defer { - var case_branch = cg.branches.pop().?; - case_branch.deinit(cg.gpa); - } - try cg.genBody(case.body); - - try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch - } - - try cg.endBlock(); - if (has_else_body) { - const else_body = cases_it.elseBody(); - - cg.branches.appendAssumeCapacity(.{}); - const else_deaths = liveness.deaths.len - 1; - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len); - defer { - var else_branch = cg.branches.pop().?; - else_branch.deinit(cg.gpa); - } - try cg.genBody(else_body); - } else { - try cg.addTag(.@"unreachable"); - } - - try cg.endBlock(); // whole switch block end - - if (is_dispatch_loop) { - try cg.endBlock(); // dispatch loop end - } - - return cg.finishAir(inst, .none, &.{}); -} - -fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br; - const switch_loop = cg.blocks.get(br.block_inst).?; - - const operand = try cg.resolveInst(br.operand); - try cg.lowerToStack(operand); - try cg.addLocal(.local_set, switch_loop.value.local.value); - - const idx: u32 = cg.block_depth - switch_loop.label; - try cg.addLabel(.br, idx); - - return cg.finishAir(inst, .none, &.{br.operand}); -} - -fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void { - const zcu = cg.pt.zcu; - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - const err_union_ty = cg.typeOf(un_op); - const pl_ty = err_union_ty.errorUnionPayload(zcu); - - const result: WValue = result: { - if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { - switch (opcode) { - .i32_ne => break :result .{ .imm32 = 0 }, - .i32_eq => break :result .{ .imm32 = 1 }, - else => unreachable, - } - } - - try cg.emitWValue(operand); - if (op_kind == .ptr or pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - try cg.addMemArg(.i32_load16_u, .{ - .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))), - .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?), - }); - } - - // Compare the error value with '0' - try cg.addImm32(0); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - break :result .stack; - }; - return cg.finishAir(inst, result, &.{un_op}); -} - -/// E!T -> T op_is_ptr == false -/// *(E!T) -> *T op_is_prt == true -fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const op_ty = cg.typeOf(ty_op.operand); - const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; - const payload_ty = eu_ty.errorUnionPayload(zcu); - - const result: WValue = result: { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - if (op_is_ptr) { - break :result cg.reuseOperand(ty_op.operand, operand); - } else { - break :result .none; - } - } - - const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); - if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) { - break :result try cg.buildPointerOffset(operand, pl_offset, .new); - } else { - assert(isByRef(eu_ty, zcu, cg.target)); - break :result try cg.load(operand, payload_ty, pl_offset); - } - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// E!T -> E op_is_ptr == false -/// *(E!T) -> E op_is_prt == true -/// NOTE: op_is_ptr will not change return type -fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const op_ty = cg.typeOf(ty_op.operand); - const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; - const payload_ty = eu_ty.errorUnionPayload(zcu); - - const result: WValue = result: { - if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { - break :result .{ .imm32 = 0 }; - } - - const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu)); - if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) { - break :result try cg.load(operand, Type.anyerror, err_offset); - } else { - assert(!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)); - break :result cg.reuseOperand(ty_op.operand, operand); - } - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const err_ty = cg.typeOfIndex(inst); - - const pl_ty = cg.typeOf(ty_op.operand); - const result = result: { - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); - } - - const err_union = try cg.allocStack(err_ty); - const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); - try cg.store(payload_ptr, operand, pl_ty, 0); - - // ensure we also write '0' to the error part, so any present stack value gets overwritten by it. - try cg.emitWValue(err_union); - try cg.addImm32(0); - const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); - try cg.addMemArg(.i32_store16, .{ - .offset = err_union.offset() + err_val_offset, - .alignment = 2, - }); - break :result err_union; - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const err_ty = ty_op.ty.toType(); - const pl_ty = err_ty.errorUnionPayload(zcu); - - const result = result: { - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); - } - - const err_union = try cg.allocStack(err_ty); - // store error value - try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu))); - - // write 'undefined' to the payload - const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); - const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu))); - try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa }); - - break :result err_union; - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airIntcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const ty = ty_op.ty.toType(); - const operand = try cg.resolveInst(ty_op.operand); - const operand_ty = cg.typeOf(ty_op.operand); - const zcu = cg.pt.zcu; - if (ty.zigTypeTag(zcu) == .vector or operand_ty.zigTypeTag(zcu) == .vector) { - return cg.fail("todo Wasm intcast for vectors", .{}); - } - if (ty.abiSize(zcu) > 16 or operand_ty.abiSize(zcu) > 16) { - return cg.fail("todo Wasm intcast for bitsize > 128", .{}); - } - - const op_bits = toWasmBits(@intCast(operand_ty.bitSize(zcu))).?; - const wanted_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?; - const result = if (op_bits == wanted_bits) - cg.reuseOperand(ty_op.operand, operand) - else - try cg.intcast(operand, operand_ty, ty); - - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// Upcasts or downcasts an integer based on the given and wanted types, -/// and stores the result in a new operand. -/// Asserts type's bitsize <= 128 -/// NOTE: May leave the result on the top of the stack. -fn intcast(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - const given_bitsize = @as(u16, @intCast(given.bitSize(zcu))); - const wanted_bitsize = @as(u16, @intCast(wanted.bitSize(zcu))); - assert(given_bitsize <= 128); - assert(wanted_bitsize <= 128); - - const op_bits = toWasmBits(given_bitsize).?; - const wanted_bits = toWasmBits(wanted_bitsize).?; - if (op_bits == wanted_bits) { - return operand; - } - - if (op_bits == 64 and wanted_bits == 32) { - try cg.emitWValue(operand); - try cg.addTag(.i32_wrap_i64); - return .stack; - } else if (op_bits == 32 and wanted_bits == 64) { - try cg.emitWValue(operand); - try cg.addTag(if (wanted.isSignedInt(zcu)) .i64_extend_i32_s else .i64_extend_i32_u); - return .stack; - } else if (wanted_bits == 128) { - // for 128bit integers we store the integer in the virtual stack, rather than a local - const stack_ptr = try cg.allocStack(wanted); - try cg.emitWValue(stack_ptr); - - // for 32 bit integers, we first coerce the value into a 64 bit integer before storing it - // meaning less store operations are required. - const lhs = if (op_bits == 32) blk: { - const sign_ty = if (wanted.isSignedInt(zcu)) Type.i64 else Type.u64; - break :blk try (try cg.intcast(operand, given, sign_ty)).toLocal(cg, sign_ty); - } else operand; - - // store lsb first - try cg.store(.stack, lhs, Type.u64, 0 + stack_ptr.offset()); - - // For signed integers we shift lsb by 63 (64bit integer - 1 sign bit) and store remaining value - if (wanted.isSignedInt(zcu)) { - try cg.emitWValue(stack_ptr); - const shr = try cg.binOp(lhs, .{ .imm64 = 63 }, Type.i64, .shr); - try cg.store(.stack, shr, Type.u64, 8 + stack_ptr.offset()); - } else { - // Ensure memory of msb is zero'd - try cg.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8); - } - return stack_ptr; - } else return cg.load(operand, wanted, 0); -} - -fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void { - const zcu = cg.pt.zcu; - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - - const op_ty = cg.typeOf(un_op); - const optional_ty = if (op_kind == .ptr) op_ty.childType(zcu) else op_ty; - const result = try cg.isNull(operand, optional_ty, opcode); - return cg.finishAir(inst, result, &.{un_op}); -} - -/// For a given type and operand, checks if it's considered `null`. -/// NOTE: Leaves the result on the stack -fn isNull(cg: *CodeGen, operand: WValue, optional_ty: Type, opcode: std.wasm.Opcode) InnerError!WValue { - const pt = cg.pt; - const zcu = pt.zcu; - try cg.emitWValue(operand); - const payload_ty = optional_ty.optionalChild(zcu); - if (!optional_ty.optionalReprIsPayload(zcu)) { - // When payload is zero-bits, we can treat operand as a value, rather than - // a pointer to the stack value - if (payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { - return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)}); - }; - try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 }); - } - } else if (payload_ty.isSlice(zcu)) { - switch (cg.ptr_size) { - .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }), - .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }), - } - } - - // Compare the null value with '0' - try cg.addImm32(0); - try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); - - return .stack; -} - -fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const opt_ty = cg.typeOf(ty_op.operand); - const payload_ty = cg.typeOfIndex(inst); - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - return cg.finishAir(inst, .none, &.{ty_op.operand}); - } - - const result = result: { - const operand = try cg.resolveInst(ty_op.operand); - if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand); - - if (isByRef(payload_ty, zcu, cg.target)) { - break :result try cg.buildPointerOffset(operand, 0, .new); - } - - break :result try cg.load(operand, payload_ty, 0); - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); - - const result = result: { - const payload_ty = opt_ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu) or opt_ty.optionalReprIsPayload(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); - } - - break :result try cg.buildPointerOffset(operand, 0, .new); - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); - const payload_ty = opt_ty.optionalChild(zcu); - - if (opt_ty.optionalReprIsPayload(zcu)) { - return cg.finishAir(inst, operand, &.{ty_op.operand}); - } - - const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { - return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)}); - }; - - try cg.emitWValue(operand); - try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 }); - - const result = try cg.buildPointerOffset(operand, 0, .new); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const payload_ty = cg.typeOf(ty_op.operand); - const pt = cg.pt; - const zcu = pt.zcu; - - const result = result: { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - const non_null_bit = try cg.allocStack(Type.u1); - try cg.emitWValue(non_null_bit); - try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 }); - break :result non_null_bit; - } - - const operand = try cg.resolveInst(ty_op.operand); - const op_ty = cg.typeOfIndex(inst); - if (op_ty.optionalReprIsPayload(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); - } - const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { - return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)}); - }; - - // Create optional type, set the non-null bit, and store the operand inside the optional type - const result_ptr = try cg.allocStack(op_ty); - try cg.emitWValue(result_ptr); - try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 }); - - const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new); - try cg.store(payload_ptr, operand, payload_ty, 0); - break :result result_ptr; - }; - - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const slice_ty = cg.typeOfIndex(inst); - - const slice = try cg.allocStack(slice_ty); - try cg.store(slice, lhs, Type.usize, 0); - try cg.store(slice, rhs, Type.usize, cg.ptrSize()); - - return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand}); -} - -fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const slice_ty = cg.typeOf(bin_op.lhs); - const slice = try cg.resolveInst(bin_op.lhs); - const index = try cg.resolveInst(bin_op.rhs); - const elem_ty = slice_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - // load pointer onto stack - _ = try cg.load(slice, Type.usize, 0); - - // calculate index into slice - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - - const elem_result = if (isByRef(elem_ty, zcu, cg.target)) - .stack - else - try cg.load(.stack, elem_ty, 0); - - return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const elem_ty = ty_pl.ty.toType().childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - const slice = try cg.resolveInst(bin_op.lhs); - const index = try cg.resolveInst(bin_op.rhs); - - _ = try cg.load(slice, Type.usize, 0); - - // calculate index into slice - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand}); -} - -fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue { - const ptr = try cg.load(operand, Type.usize, 0); - return ptr.toLocal(cg, Type.usize); -} - -fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue { - const len = try cg.load(operand, Type.usize, cg.ptrSize()); - return len.toLocal(cg, Type.usize); -} - -fn airTrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const wanted_ty: Type = ty_op.ty.toType(); - const op_ty = cg.typeOf(ty_op.operand); - const zcu = cg.pt.zcu; - - if (wanted_ty.zigTypeTag(zcu) == .vector or op_ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: trunc for vectors", .{}); - } - - const result = if (op_ty.bitSize(zcu) == wanted_ty.bitSize(zcu)) - cg.reuseOperand(ty_op.operand, operand) - else - try cg.trunc(operand, wanted_ty, op_ty); - - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// Truncates a given operand to a given type, discarding any overflown bits. -/// NOTE: Resulting value is left on the stack. -fn trunc(cg: *CodeGen, operand: WValue, wanted_ty: Type, given_ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - const given_bits = @as(u16, @intCast(given_ty.bitSize(zcu))); - if (toWasmBits(given_bits) == null) { - return cg.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{given_bits}); - } - - var result = try cg.intcast(operand, given_ty, wanted_ty); - const wanted_bits = @as(u16, @intCast(wanted_ty.bitSize(zcu))); - const wasm_bits = toWasmBits(wanted_bits).?; - if (wasm_bits != wanted_bits) { - result = try cg.wrapOperand(result, wanted_ty); - } - return result; -} - -fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const array_ty = cg.typeOf(ty_op.operand).childType(zcu); - const slice_ty = ty_op.ty.toType(); - - // create a slice on the stack - const slice_local = try cg.allocStack(slice_ty); - - // store the array ptr in the slice - if (array_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - try cg.store(slice_local, operand, Type.usize, 0); - } - - // store the length of the array in the slice - const array_len: u32 = @intCast(array_ty.arrayLen(zcu)); - try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize()); - - return cg.finishAir(inst, slice_local, &.{ty_op.operand}); -} - -fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ptr_ty = cg.typeOf(bin_op.lhs); - const ptr = try cg.resolveInst(bin_op.lhs); - const index = try cg.resolveInst(bin_op.rhs); - const elem_ty = ptr_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - // load pointer onto the stack - if (ptr_ty.isSlice(zcu)) { - _ = try cg.load(ptr, Type.usize, 0); - } else { - try cg.lowerToStack(ptr); - } - - // calculate index into slice - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - - const elem_result = if (isByRef(elem_ty, zcu, cg.target)) - .stack - else - try cg.load(.stack, elem_ty, 0); - - return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const ptr_ty = cg.typeOf(bin_op.lhs); - const elem_ty = ty_pl.ty.toType().childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - const ptr = try cg.resolveInst(bin_op.lhs); - const index = try cg.resolveInst(bin_op.rhs); - - // load pointer onto the stack - if (ptr_ty.isSlice(zcu)) { - _ = try cg.load(ptr, Type.usize, 0); - } else { - try cg.lowerToStack(ptr); - } - - // calculate index into ptr - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { - const zcu = cg.pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const ptr = try cg.resolveInst(bin_op.lhs); - const offset = try cg.resolveInst(bin_op.rhs); - const ptr_ty = cg.typeOf(bin_op.lhs); - const pointee_ty = switch (ptr_ty.ptrSize(zcu)) { - .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type - else => ptr_ty.childType(zcu), - }; - - const valtype = typeToValtype(Type.usize, zcu, cg.target); - const mul_opcode = buildOpcode(.{ .valtype1 = valtype, .op = .mul }); - const bin_opcode = buildOpcode(.{ .valtype1 = valtype, .op = op }); - - try cg.lowerToStack(ptr); - try cg.emitWValue(offset); - try cg.addImm32(@intCast(pointee_ty.abiSize(zcu))); - try cg.addTag(Mir.Inst.Tag.fromOpcode(mul_opcode)); - try cg.addTag(Mir.Inst.Tag.fromOpcode(bin_opcode)); - - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ptr = try cg.resolveInst(bin_op.lhs); - const ptr_ty = cg.typeOf(bin_op.lhs); - const value = try cg.resolveInst(bin_op.rhs); - const len = switch (ptr_ty.ptrSize(zcu)) { - .slice => try cg.sliceLen(ptr), - .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }), - .c, .many => unreachable, - }; - - const elem_ty = if (ptr_ty.ptrSize(zcu) == .one) - ptr_ty.childType(zcu).childType(zcu) - else - ptr_ty.childType(zcu); - - if (!safety and bin_op.rhs == .undef) { - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); - } - - const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty); - try cg.memset(elem_ty, dst_ptr, len, value); - - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); -} - -/// Sets a region of memory at `ptr` to the value of `value` -/// When the user has enabled the bulk_memory feature, we lower -/// this to wasm's memset instruction. When the feature is not present, -/// we implement it manually. -fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void { - const zcu = cg.pt.zcu; - const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); - - // When bulk_memory is enabled, we lower it to wasm's memset instruction. - // If not, we lower it ourselves. - if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) { - const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0); - - if (!len0_ok) { - try cg.startBlock(.block, .empty); - - // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is - // out of memory bounds. This can easily happen in Zig in a case such as: - // - // const ptr: [*]u8 = undefined; - // var len: usize = runtime_zero(); - // @memset(ptr[0..len], 42); - // - // So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design. - try cg.emitWValue(len); - try cg.addTag(.i32_eqz); - try cg.addLabel(.br_if, 0); - } - - try cg.lowerToStack(ptr); - try cg.emitWValue(value); - try cg.emitWValue(len); - try cg.addExtended(.memory_fill); - - if (!len0_ok) { - try cg.endBlock(); - } - - return; - } - - const final_len: WValue = switch (len) { - .imm32 => |val| .{ .imm32 = val * abi_size }, - .imm64 => |val| .{ .imm64 = val * abi_size }, - else => if (abi_size != 1) blk: { - const new_len = try cg.ensureAllocLocal(Type.usize); - try cg.emitWValue(len); - switch (cg.ptr_size) { - .wasm32 => { - try cg.emitWValue(.{ .imm32 = abi_size }); - try cg.addTag(.i32_mul); - }, - .wasm64 => { - try cg.emitWValue(.{ .imm64 = abi_size }); - try cg.addTag(.i64_mul); - }, - } - try cg.addLocal(.local_set, new_len.local.value); - break :blk new_len; - } else len, - }; - - var end_ptr = try cg.allocLocal(Type.usize); - defer end_ptr.free(cg); - var new_ptr = try cg.buildPointerOffset(ptr, 0, .new); - defer new_ptr.free(cg); - - // get the loop conditional: if current pointer address equals final pointer's address - try cg.lowerToStack(ptr); - try cg.emitWValue(final_len); - switch (cg.ptr_size) { - .wasm32 => try cg.addTag(.i32_add), - .wasm64 => try cg.addTag(.i64_add), - } - try cg.addLocal(.local_set, end_ptr.local.value); - - // outer block to jump to when loop is done - try cg.startBlock(.block, .empty); - try cg.startBlock(.loop, .empty); - - // check for condition for loop end - try cg.emitWValue(new_ptr); - try cg.emitWValue(end_ptr); - switch (cg.ptr_size) { - .wasm32 => try cg.addTag(.i32_eq), - .wasm64 => try cg.addTag(.i64_eq), - } - try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished) - - // store the value at the current position of the pointer - try cg.store(new_ptr, value, elem_ty, 0); - - // move the pointer to the next element - try cg.emitWValue(new_ptr); - switch (cg.ptr_size) { - .wasm32 => { - try cg.emitWValue(.{ .imm32 = abi_size }); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.emitWValue(.{ .imm64 = abi_size }); - try cg.addTag(.i64_add); - }, - } - try cg.addLocal(.local_set, new_ptr.local.value); - - // end of loop - try cg.addLabel(.br, 0); // jump to start of loop - try cg.endBlock(); - try cg.endBlock(); -} - -fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const array_ty = cg.typeOf(bin_op.lhs); - const array = try cg.resolveInst(bin_op.lhs); - const index = try cg.resolveInst(bin_op.rhs); - const elem_ty = array_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - if (isByRef(array_ty, zcu, cg.target)) { - try cg.lowerToStack(array); - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - } else { - assert(array_ty.zigTypeTag(zcu) == .vector); - - switch (index) { - inline .imm32, .imm64 => |lane| { - const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) { - 8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u, - 16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u, - 32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane, - 64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane, - else => unreachable, - }; - - var operands = [_]u32{ @intFromEnum(opcode), @as(u8, @intCast(lane)) }; - - try cg.emitWValue(array); - - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - try cg.mir_extra.appendSlice(cg.gpa, &operands); - try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); - - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); - }, - else => { - const stack_vec = try cg.allocStack(array_ty); - try cg.store(stack_vec, array, array_ty, 0); - - // Is a non-unrolled vector (v128) - try cg.lowerToStack(stack_vec); - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - }, - } - } - - const elem_result = if (isByRef(elem_ty, zcu, cg.target)) - .stack - else - try cg.load(.stack, elem_ty, 0); - - return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airIntFromFloat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const op_ty = cg.typeOf(ty_op.operand); - const op_bits = op_ty.floatBits(cg.target); - - const dest_ty = cg.typeOfIndex(inst); - const dest_info = dest_ty.intInfo(zcu); - - if (dest_info.bits > 128) { - return cg.fail("TODO: intFromFloat for integers/floats with bitsize {}", .{dest_info.bits}); - } - - if ((op_bits != 32 and op_bits != 64) or dest_info.bits > 64) { - const dest_bitsize = if (dest_info.bits <= 32) 32 else std.math.ceilPowerOfTwoAssert(u16, dest_info.bits); - - const intrinsic = switch (dest_info.signedness) { - inline .signed, .unsigned => |ct_s| switch (op_bits) { - inline 16, 32, 64, 80, 128 => |ct_op_bits| switch (dest_bitsize) { - inline 32, 64, 128 => |ct_dest_bits| @field( - Mir.Intrinsic, - "__fix" ++ switch (ct_s) { - .signed => "", - .unsigned => "uns", - } ++ - compilerRtFloatAbbrev(ct_op_bits) ++ "f" ++ - compilerRtIntAbbrev(ct_dest_bits) ++ "i", - ), - else => unreachable, - }, - else => unreachable, - }, - }; - const result = try cg.callIntrinsic(intrinsic, &.{op_ty.ip_index}, dest_ty, &.{operand}); - return cg.finishAir(inst, result, &.{ty_op.operand}); - } - - try cg.emitWValue(operand); - const op = buildOpcode(.{ - .op = .trunc, - .valtype1 = typeToValtype(dest_ty, zcu, cg.target), - .valtype2 = typeToValtype(op_ty, zcu, cg.target), - .signedness = dest_info.signedness, - }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(op)); - const result = try cg.wrapOperand(.stack, dest_ty); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airFloatFromInt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const op_ty = cg.typeOf(ty_op.operand); - const op_info = op_ty.intInfo(zcu); - - const dest_ty = cg.typeOfIndex(inst); - const dest_bits = dest_ty.floatBits(cg.target); - - if (op_info.bits > 128) { - return cg.fail("TODO: floatFromInt for integers/floats with bitsize {d} bits", .{op_info.bits}); - } - - if (op_info.bits > 64 or (dest_bits > 64 or dest_bits < 32)) { - const op_bitsize = if (op_info.bits <= 32) 32 else std.math.ceilPowerOfTwoAssert(u16, op_info.bits); - - const intrinsic = switch (op_info.signedness) { - inline .signed, .unsigned => |ct_s| switch (op_bitsize) { - inline 32, 64, 128 => |ct_int_bits| switch (dest_bits) { - inline 16, 32, 64, 80, 128 => |ct_float_bits| @field( - Mir.Intrinsic, - "__float" ++ switch (ct_s) { - .signed => "", - .unsigned => "un", - } ++ - compilerRtIntAbbrev(ct_int_bits) ++ "i" ++ - compilerRtFloatAbbrev(ct_float_bits) ++ "f", - ), - else => unreachable, - }, - else => unreachable, - }, - }; - - const result = try cg.callIntrinsic(intrinsic, &.{op_ty.ip_index}, dest_ty, &.{operand}); - return cg.finishAir(inst, result, &.{ty_op.operand}); - } - - try cg.emitWValue(operand); - const op = buildOpcode(.{ - .op = .convert, - .valtype1 = typeToValtype(dest_ty, zcu, cg.target), - .valtype2 = typeToValtype(op_ty, zcu, cg.target), - .signedness = op_info.signedness, - }); - try cg.addTag(Mir.Inst.Tag.fromOpcode(op)); - - return cg.finishAir(inst, .stack, &.{ty_op.operand}); -} - -fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try cg.resolveInst(ty_op.operand); - const ty = cg.typeOfIndex(inst); - const elem_ty = ty.childType(zcu); - - if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: { - switch (operand) { - // when the operand lives in the linear memory section, we can directly - // load and splat the value at once. Meaning we do not first have to load - // the scalar value onto the stack. - .stack_offset, .nav_ref, .uav_ref => { - const opcode = switch (elem_ty.bitSize(zcu)) { - 8 => @intFromEnum(std.wasm.SimdOpcode.v128_load8_splat), - 16 => @intFromEnum(std.wasm.SimdOpcode.v128_load16_splat), - 32 => @intFromEnum(std.wasm.SimdOpcode.v128_load32_splat), - 64 => @intFromEnum(std.wasm.SimdOpcode.v128_load64_splat), - else => break :blk, // Cannot make use of simd-instructions - }; - try cg.emitWValue(operand); - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - // stores as := opcode, offset, alignment (opcode::memarg) - try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ - opcode, - operand.offset(), - @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?), - }); - try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); - return cg.finishAir(inst, .stack, &.{ty_op.operand}); - }, - .local => { - const opcode = switch (elem_ty.bitSize(zcu)) { - 8 => @intFromEnum(std.wasm.SimdOpcode.i8x16_splat), - 16 => @intFromEnum(std.wasm.SimdOpcode.i16x8_splat), - 32 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i32x4_splat) else @intFromEnum(std.wasm.SimdOpcode.f32x4_splat), - 64 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i64x2_splat) else @intFromEnum(std.wasm.SimdOpcode.f64x2_splat), - else => break :blk, // Cannot make use of simd-instructions - }; - try cg.emitWValue(operand); - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - try cg.mir_extra.append(cg.gpa, opcode); - try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); - return cg.finishAir(inst, .stack, &.{ty_op.operand}); - }, - else => unreachable, - } - } - const elem_size = elem_ty.bitSize(zcu); - const vector_len = @as(usize, @intCast(ty.vectorLen(zcu))); - if ((!std.math.isPowerOfTwo(elem_size) or elem_size % 8 != 0) and vector_len > 1) { - return cg.fail("TODO: WebAssembly `@splat` for arbitrary element bitsize {d}", .{elem_size}); - } - - const result = try cg.allocStack(ty); - const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); - var index: usize = 0; - var offset: u32 = 0; - while (index < vector_len) : (index += 1) { - try cg.store(result, operand, elem_ty, offset); - offset += elem_byte_size; - } - - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const operand = try cg.resolveInst(pl_op.operand); - - _ = operand; - return cg.fail("TODO: Implement wasm airSelect", .{}); -} - -fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - - const unwrapped = cg.air.unwrapShuffleOne(zcu, inst); - const result_ty = unwrapped.result_ty; - const mask = unwrapped.mask; - const operand = try cg.resolveInst(unwrapped.operand); - - const elem_ty = result_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - // TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were - // to lower the comptime-known operands to a non-by-ref vector value. - - // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible. - // I tried to fix it, but I couldn't make much sense of how this backend handles memory. - if (!isByRef(result_ty, zcu, cg.target) or - !isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{}); - - const dest_alloc = try cg.allocStack(result_ty); - for (mask, 0..) |mask_elem, out_idx| { - try cg.emitWValue(dest_alloc); - const elem_val = switch (mask_elem.unwrap()) { - .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)), - .value => |val| try cg.lowerConstant(.fromInterned(val), elem_ty), - }; - try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx)); - } - return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand}); -} - -fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - - const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst); - const result_ty = unwrapped.result_ty; - const mask = unwrapped.mask; - const operand_a = try cg.resolveInst(unwrapped.operand_a); - const operand_b = try cg.resolveInst(unwrapped.operand_b); - - const a_ty = cg.typeOf(unwrapped.operand_a); - const b_ty = cg.typeOf(unwrapped.operand_b); - const elem_ty = result_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - - // WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8 - // and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise, - // we fall back to a naive loop lowering. - if (!isByRef(a_ty, zcu, cg.target) and - !isByRef(b_ty, zcu, cg.target) and - !isByRef(result_ty, zcu, cg.target) and - elem_ty.bitSize(zcu) % 8 == 0) - { - var lane_map: [16]u8 align(4) = undefined; - const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8); - for (mask, 0..) |mask_elem, out_idx| { - const out_first_lane = out_idx * lanes_per_elem; - const in_first_lane = switch (mask_elem.unwrap()) { - .a_elem => |i| i * lanes_per_elem, - .b_elem => |i| i * lanes_per_elem + 16, - .undef => 0, // doesn't matter - }; - for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| { - out.* = @intCast(in); - } - } - try cg.emitWValue(operand_a); - try cg.emitWValue(operand_b); - const extra_index: u32 = @intCast(cg.mir_extra.items.len); - try cg.mir_extra.appendSlice(cg.gpa, &.{ - @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle), - @bitCast(lane_map[0..4].*), - @bitCast(lane_map[4..8].*), - @bitCast(lane_map[8..12].*), - @bitCast(lane_map[12..].*), - }); - try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); - return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b }); - } - - // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible. - // I tried to fix it, but I couldn't make much sense of how this backend handles memory. - if (!isByRef(result_ty, zcu, cg.target) or - !isByRef(a_ty, zcu, cg.target) or - !isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{}); - - const dest_alloc = try cg.allocStack(result_ty); - for (mask, 0..) |mask_elem, out_idx| { - try cg.emitWValue(dest_alloc); - const elem_val = switch (mask_elem.unwrap()) { - .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)), - .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)), - .undef => try cg.emitUndefined(elem_ty), - }; - try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx)); - } - return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b }); -} - -fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const reduce = cg.air.instructions.items(.data)[@intFromEnum(inst)].reduce; - const operand = try cg.resolveInst(reduce.operand); - - _ = operand; - return cg.fail("TODO: Implement wasm airReduce", .{}); -} - -fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const result_ty = cg.typeOfIndex(inst); - const len = @as(usize, @intCast(result_ty.arrayLen(zcu))); - const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]); - - const result: WValue = result_value: { - switch (result_ty.zigTypeTag(zcu)) { - .array => { - const result = try cg.allocStack(result_ty); - const elem_ty = result_ty.childType(zcu); - const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); - const sentinel = result_ty.sentinel(zcu); - - // When the element type is by reference, we must copy the entire - // value. It is therefore safer to move the offset pointer and store - // each value individually, instead of using store offsets. - if (isByRef(elem_ty, zcu, cg.target)) { - // copy stack pointer into a temporary local, which is - // moved for each element to store each value in the right position. - const offset = try cg.buildPointerOffset(result, 0, .new); - for (elements, 0..) |elem, elem_index| { - const elem_val = try cg.resolveInst(elem); - try cg.store(offset, elem_val, elem_ty, 0); - - if (elem_index < elements.len - 1 or sentinel != null) { - _ = try cg.buildPointerOffset(offset, elem_size, .modify); - } - } - if (sentinel) |s| { - const val = try cg.resolveValue(s); - try cg.store(offset, val, elem_ty, 0); - } - } else { - var offset: u32 = 0; - for (elements) |elem| { - const elem_val = try cg.resolveInst(elem); - try cg.store(result, elem_val, elem_ty, offset); - offset += elem_size; - } - if (sentinel) |s| { - const val = try cg.resolveValue(s); - try cg.store(result, val, elem_ty, offset); - } - } - break :result_value result; - }, - .@"struct" => switch (result_ty.containerLayout(zcu)) { - .@"packed" => { - if (isByRef(result_ty, zcu, cg.target)) { - return cg.fail("TODO: airAggregateInit for packed structs larger than 64 bits", .{}); - } - const packed_struct = zcu.typeToPackedStruct(result_ty).?; - const field_types = packed_struct.field_types; - const backing_type = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); - - // ensure the result is zero'd - const result = try cg.allocLocal(backing_type); - if (backing_type.bitSize(zcu) <= 32) - try cg.addImm32(0) - else - try cg.addImm64(0); - try cg.addLocal(.local_set, result.local.value); - - var current_bit: u16 = 0; - for (elements, 0..) |elem, elem_index| { - const field_ty = Type.fromInterned(field_types.get(ip)[elem_index]); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - - const shift_val: WValue = if (backing_type.bitSize(zcu) <= 32) - .{ .imm32 = current_bit } - else - .{ .imm64 = current_bit }; - - const value = try cg.resolveInst(elem); - const value_bit_size: u16 = @intCast(field_ty.bitSize(zcu)); - const int_ty = try pt.intType(.unsigned, value_bit_size); - - // load our current result on stack so we can perform all transformations - // using only stack values. Saving the cost of loads and stores. - try cg.emitWValue(result); - const bitcasted = try cg.bitcast(int_ty, field_ty, value); - const extended_val = try cg.intcast(bitcasted, int_ty, backing_type); - // no need to shift any values when the current offset is 0 - const shifted = if (current_bit != 0) shifted: { - break :shifted try cg.binOp(extended_val, shift_val, backing_type, .shl); - } else extended_val; - // we ignore the result as we keep it on the stack to assign it directly to `result` - _ = try cg.binOp(.stack, shifted, backing_type, .@"or"); - try cg.addLocal(.local_set, result.local.value); - current_bit += value_bit_size; - } - break :result_value result; - }, - else => { - const result = try cg.allocStack(result_ty); - const offset = try cg.buildPointerOffset(result, 0, .new); // pointer to offset - var prev_field_offset: u64 = 0; - for (elements, 0..) |elem, elem_index| { - if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue; - - const elem_ty = result_ty.fieldType(elem_index, zcu); - const field_offset = result_ty.structFieldOffset(elem_index, zcu); - _ = try cg.buildPointerOffset(offset, @intCast(field_offset - prev_field_offset), .modify); - prev_field_offset = field_offset; - - const value = try cg.resolveInst(elem); - try cg.store(offset, value, elem_ty, 0); - } - - break :result_value result; - }, - }, - .vector => return cg.fail("TODO: Wasm backend: implement airAggregateInit for vectors", .{}), - else => unreachable, - } - }; - - if (elements.len <= Air.Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); - @memcpy(buf[0..elements.len], elements); - return cg.finishAir(inst, result, &buf); - } - var bt = try cg.iterateBigTomb(inst, elements.len); - for (elements) |arg| bt.feed(arg); - return bt.finishAir(result); -} - -fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; - - const result = result: { - const union_ty = cg.typeOfIndex(inst); - const layout = union_ty.unionGetLayout(zcu); - const union_obj = zcu.typeToUnion(union_ty).?; - const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]); - const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index]; - - const tag_int = blk: { - const tag_ty = union_ty.unionTagTypeHypothetical(zcu); - const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?; - const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index); - break :blk try cg.lowerConstant(tag_val, tag_ty); - }; - if (layout.payload_size == 0) { - if (layout.tag_size == 0) { - break :result .none; - } - assert(!isByRef(union_ty, zcu, cg.target)); - break :result tag_int; - } - - if (isByRef(union_ty, zcu, cg.target)) { - const result_ptr = try cg.allocStack(union_ty); - const payload = try cg.resolveInst(extra.init); - if (layout.tag_align.compare(.gte, layout.payload_align)) { - if (isByRef(field_ty, zcu, cg.target)) { - const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new); - try cg.store(payload_ptr, payload, field_ty, 0); - } else { - try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size)); - } - - if (layout.tag_size > 0) { - try cg.store(result_ptr, tag_int, Type.fromInterned(union_obj.enum_tag_ty), 0); - } - } else { - try cg.store(result_ptr, payload, field_ty, 0); - if (layout.tag_size > 0) { - try cg.store( - result_ptr, - tag_int, - Type.fromInterned(union_obj.enum_tag_ty), - @intCast(layout.payload_size), - ); - } - } - break :result result_ptr; - } else { - const operand = try cg.resolveInst(extra.init); - const union_int_type = try pt.intType(.unsigned, @as(u16, @intCast(union_ty.bitSize(zcu)))); - if (field_ty.zigTypeTag(zcu) == .float) { - const int_type = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu))); - const bitcasted = try cg.bitcast(field_ty, int_type, operand); - break :result try cg.trunc(bitcasted, int_type, union_int_type); - } else if (field_ty.isPtrAtRuntime(zcu)) { - const int_type = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu))); - break :result try cg.intcast(operand, int_type, union_int_type); - } - break :result try cg.intcast(operand, field_ty, union_int_type); - } - }; - - return cg.finishAir(inst, result, &.{extra.init}); -} - -fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const prefetch = cg.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; - return cg.finishAir(inst, .none, &.{prefetch.ptr}); -} - -fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - - try cg.addLabel(.memory_size, pl_op.payload); - return cg.finishAir(inst, .stack, &.{pl_op.operand}); -} - -fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void { - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - - const operand = try cg.resolveInst(pl_op.operand); - try cg.emitWValue(operand); - try cg.addLabel(.memory_grow, pl_op.payload); - return cg.finishAir(inst, .stack, &.{pl_op.operand}); -} - -fn cmpOptionals(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue { - const zcu = cg.pt.zcu; - assert(operand_ty.hasRuntimeBitsIgnoreComptime(zcu)); - assert(op == .eq or op == .neq); - const payload_ty = operand_ty.optionalChild(zcu); - assert(!isByRef(payload_ty, zcu, cg.target)); - - var result = try cg.allocLocal(Type.i32); - defer result.free(cg); - - var lhs_null = try cg.allocLocal(Type.i32); - defer lhs_null.free(cg); - - try cg.startBlock(.block, .empty); - - try cg.addImm32(if (op == .eq) 0 else 1); - try cg.addLocal(.local_set, result.local.value); - - _ = try cg.isNull(lhs, operand_ty, .i32_eq); - try cg.addLocal(.local_tee, lhs_null.local.value); - _ = try cg.isNull(rhs, operand_ty, .i32_eq); - try cg.addTag(.i32_ne); - try cg.addLabel(.br_if, 0); // only one is null - - try cg.addImm32(if (op == .eq) 1 else 0); - try cg.addLocal(.local_set, result.local.value); - - try cg.addLocal(.local_get, lhs_null.local.value); - try cg.addLabel(.br_if, 0); // both are null - - _ = try cg.load(lhs, payload_ty, 0); - _ = try cg.load(rhs, payload_ty, 0); - _ = try cg.cmp(.stack, .stack, payload_ty, op); - try cg.addLocal(.local_set, result.local.value); - - try cg.endBlock(); - - try cg.addLocal(.local_get, result.local.value); - - return .stack; -} - -/// Compares big integers by checking both its high bits and low bits. -/// NOTE: Leaves the result of the comparison on top of the stack. -/// TODO: Lower this to compiler_rt call when bitsize > 128 -fn cmpBigInt(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue { - const zcu = cg.pt.zcu; - assert(operand_ty.abiSize(zcu) >= 16); - assert(!(lhs != .stack and rhs == .stack)); - if (operand_ty.bitSize(zcu) > 128) { - return cg.fail("TODO: Support cmpBigInt for integer bitsize: '{d}'", .{operand_ty.bitSize(zcu)}); - } - - var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64); - defer lhs_msb.free(cg); - var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64); - defer rhs_msb.free(cg); - - switch (op) { - .eq, .neq => { - const xor_high = try cg.binOp(lhs_msb, rhs_msb, Type.u64, .xor); - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); - const xor_low = try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, .xor); - const or_result = try cg.binOp(xor_high, xor_low, Type.u64, .@"or"); - - switch (op) { - .eq => return cg.cmp(or_result, .{ .imm64 = 0 }, Type.u64, .eq), - .neq => return cg.cmp(or_result, .{ .imm64 = 0 }, Type.u64, .neq), - else => unreachable, - } - }, - else => { - const ty = if (operand_ty.isSignedInt(zcu)) Type.i64 else Type.u64; - // leave those value on top of the stack for '.select' - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); - _ = try cg.cmp(lhs_lsb, rhs_lsb, Type.u64, op); - _ = try cg.cmp(lhs_msb, rhs_msb, ty, op); - _ = try cg.cmp(lhs_msb, rhs_msb, ty, .eq); - try cg.addTag(.select); - }, - } - - return .stack; -} - -fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const un_ty = cg.typeOf(bin_op.lhs).childType(zcu); - const tag_ty = cg.typeOf(bin_op.rhs); - const layout = un_ty.unionGetLayout(zcu); - if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); - - const union_ptr = try cg.resolveInst(bin_op.lhs); - const new_tag = try cg.resolveInst(bin_op.rhs); - if (layout.payload_size == 0) { - try cg.store(union_ptr, new_tag, tag_ty, 0); - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); - } - - // when the tag alignment is smaller than the payload, the field will be stored - // after the payload. - const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: { - break :blk @intCast(layout.payload_size); - } else 0; - try cg.store(union_ptr, new_tag, tag_ty, offset); - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const un_ty = cg.typeOf(ty_op.operand); - const tag_ty = cg.typeOfIndex(inst); - const layout = un_ty.unionGetLayout(zcu); - if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand}); - - const operand = try cg.resolveInst(ty_op.operand); - // when the tag alignment is smaller than the payload, the field will be stored - // after the payload. - const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) - @intCast(layout.payload_size) - else - 0; - const result = try cg.load(operand, tag_ty, offset); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airFpext(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dest_ty = cg.typeOfIndex(inst); - const operand = try cg.resolveInst(ty_op.operand); - const result = try cg.fpext(operand, cg.typeOf(ty_op.operand), dest_ty); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// Extends a float from a given `Type` to a larger wanted `Type`, leaving the -/// result on the stack. -fn fpext(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue { - const given_bits = given.floatBits(cg.target); - const wanted_bits = wanted.floatBits(cg.target); - - const intrinsic: Mir.Intrinsic = switch (given_bits) { - 16 => switch (wanted_bits) { - 32 => { - assert(.stack == try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand})); - return .stack; - }, - 64 => { - assert(.stack == try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand})); - try cg.addTag(.f64_promote_f32); - return .stack; - }, - 80 => .__extendhfxf2, - 128 => .__extendhftf2, - else => unreachable, - }, - 32 => switch (wanted_bits) { - 64 => { - try cg.emitWValue(operand); - try cg.addTag(.f64_promote_f32); - return .stack; - }, - 80 => .__extendsfxf2, - 128 => .__extendsftf2, - else => unreachable, - }, - 64 => switch (wanted_bits) { - 80 => .__extenddfxf2, - 128 => .__extenddftf2, - else => unreachable, - }, - 80 => switch (wanted_bits) { - 128 => .__extendxftf2, - else => unreachable, - }, - else => unreachable, - }; - return cg.callIntrinsic(intrinsic, &.{given.ip_index}, wanted, &.{operand}); -} - -fn airFptrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dest_ty = cg.typeOfIndex(inst); - const operand = try cg.resolveInst(ty_op.operand); - const result = try cg.fptrunc(operand, cg.typeOf(ty_op.operand), dest_ty); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// Truncates a float from a given `Type` to its wanted `Type`, leaving the -/// result on the stack. -fn fptrunc(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue { - const given_bits = given.floatBits(cg.target); - const wanted_bits = wanted.floatBits(cg.target); - - const intrinsic: Mir.Intrinsic = switch (given_bits) { - 32 => switch (wanted_bits) { - 16 => { - return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand}); - }, - else => unreachable, - }, - 64 => switch (wanted_bits) { - 16 => { - try cg.emitWValue(operand); - try cg.addTag(.f32_demote_f64); - return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack}); - }, - 32 => { - try cg.emitWValue(operand); - try cg.addTag(.f32_demote_f64); - return .stack; - }, - else => unreachable, - }, - 80 => switch (wanted_bits) { - 16 => .__truncxfhf2, - 32 => .__truncxfsf2, - 64 => .__truncxfdf2, - else => unreachable, - }, - 128 => switch (wanted_bits) { - 16 => .__trunctfhf2, - 32 => .__trunctfsf2, - 64 => .__trunctfdf2, - 80 => .__trunctfxf2, - else => unreachable, - }, - else => unreachable, - }; - return cg.callIntrinsic(intrinsic, &.{given.ip_index}, wanted, &.{operand}); -} - -fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu); - const payload_ty = err_set_ty.errorUnionPayload(zcu); - const operand = try cg.resolveInst(ty_op.operand); - - // set error-tag to '0' to annotate error union is non-error - try cg.store( - operand, - .{ .imm32 = 0 }, - Type.anyerror, - @intCast(errUnionErrorOffset(payload_ty, zcu)), - ); - - const result = result: { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); - } - - break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new); - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; - - const field_ptr = try cg.resolveInst(extra.field_ptr); - const parent_ptr_ty = cg.typeOfIndex(inst); - const parent_ty = parent_ptr_ty.childType(zcu); - const field_ptr_ty = cg.typeOf(extra.field_ptr); - const field_index = extra.field_index; - const field_offset = switch (parent_ty.containerLayout(zcu)) { - .auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu), - .@"packed" => offset: { - const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset; - const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0; - const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset; - break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8); - }, - }; - - const result = if (field_offset != 0) result: { - const base = try cg.buildPointerOffset(field_ptr, 0, .new); - try cg.addLocal(.local_get, base.local.value); - try cg.addImm32(@intCast(field_offset)); - try cg.addTag(.i32_sub); - try cg.addLocal(.local_set, base.local.value); - break :result base; - } else cg.reuseOperand(extra.field_ptr, field_ptr); - - return cg.finishAir(inst, result, &.{extra.field_ptr}); -} - -fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - if (ptr_ty.isSlice(zcu)) { - return cg.slicePtr(ptr); - } else { - return ptr; - } -} - -fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dst = try cg.resolveInst(bin_op.lhs); - const dst_ty = cg.typeOf(bin_op.lhs); - const ptr_elem_ty = dst_ty.childType(zcu); - const src = try cg.resolveInst(bin_op.rhs); - const src_ty = cg.typeOf(bin_op.rhs); - const len = switch (dst_ty.ptrSize(zcu)) { - .slice => blk: { - const slice_len = try cg.sliceLen(dst); - if (ptr_elem_ty.abiSize(zcu) != 1) { - try cg.emitWValue(slice_len); - try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) }); - try cg.addTag(.i32_mul); - try cg.addLocal(.local_set, slice_len.local.value); - } - break :blk slice_len; - }, - .one => @as(WValue, .{ - .imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))), - }), - .c, .many => unreachable, - }; - const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty); - const src_ptr = try cg.sliceOrArrayPtr(src, src_ty); - try cg.memcpy(dst_ptr, src_ptr, len); - - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - // TODO: Implement this properly once stack serialization is solved - return cg.finishAir(inst, switch (cg.ptr_size) { - .wasm32 => .{ .imm32 = 0 }, - .wasm64 => .{ .imm64 = 0 }, - }, &.{}); -} - -fn airPopcount(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const op_ty = cg.typeOf(ty_op.operand); - - if (op_ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: Implement @popCount for vectors", .{}); - } - - const int_info = op_ty.intInfo(zcu); - const bits = int_info.bits; - const wasm_bits = toWasmBits(bits) orelse { - return cg.fail("TODO: Implement @popCount for integers with bitsize '{d}'", .{bits}); - }; - - switch (wasm_bits) { - 32 => { - try cg.emitWValue(operand); - if (op_ty.isSignedInt(zcu) and bits != wasm_bits) { - _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits)); - } - try cg.addTag(.i32_popcnt); - }, - 64 => { - try cg.emitWValue(operand); - if (op_ty.isSignedInt(zcu) and bits != wasm_bits) { - _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits)); - } - try cg.addTag(.i64_popcnt); - try cg.addTag(.i32_wrap_i64); - try cg.emitWValue(operand); - }, - 128 => { - _ = try cg.load(operand, Type.u64, 0); - try cg.addTag(.i64_popcnt); - _ = try cg.load(operand, Type.u64, 8); - if (op_ty.isSignedInt(zcu) and bits != wasm_bits) { - _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits - 64)); - } - try cg.addTag(.i64_popcnt); - try cg.addTag(.i64_add); - try cg.addTag(.i32_wrap_i64); - }, - else => unreachable, - } - - return cg.finishAir(inst, .stack, &.{ty_op.operand}); -} - -fn airBitReverse(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const ty = cg.typeOf(ty_op.operand); - - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: Implement @bitReverse for vectors", .{}); - } - - const int_info = ty.intInfo(zcu); - const bits = int_info.bits; - const wasm_bits = toWasmBits(bits) orelse { - return cg.fail("TODO: Implement @bitReverse for integers with bitsize '{d}'", .{bits}); - }; - - switch (wasm_bits) { - 32 => { - const intrin_ret = try cg.callIntrinsic( - .__bitreversesi2, - &.{.u32_type}, - Type.u32, - &.{operand}, - ); - const result = if (bits == 32) - intrin_ret - else - try cg.binOp(intrin_ret, .{ .imm32 = 32 - bits }, ty, .shr); - return cg.finishAir(inst, result, &.{ty_op.operand}); - }, - 64 => { - const intrin_ret = try cg.callIntrinsic( - .__bitreversedi2, - &.{.u64_type}, - Type.u64, - &.{operand}, - ); - const result = if (bits == 64) - intrin_ret - else - try cg.binOp(intrin_ret, .{ .imm64 = 64 - bits }, ty, .shr); - return cg.finishAir(inst, result, &.{ty_op.operand}); - }, - 128 => { - const result = try cg.allocStack(ty); - - try cg.emitWValue(result); - const first_half = try cg.load(operand, Type.u64, 8); - const intrin_ret_first = try cg.callIntrinsic( - .__bitreversedi2, - &.{.u64_type}, - Type.u64, - &.{first_half}, - ); - try cg.emitWValue(intrin_ret_first); - if (bits < 128) { - try cg.emitWValue(.{ .imm64 = 128 - bits }); - try cg.addTag(.i64_shr_u); - } - try cg.emitWValue(result); - const second_half = try cg.load(operand, Type.u64, 0); - const intrin_ret_second = try cg.callIntrinsic( - .__bitreversedi2, - &.{.u64_type}, - Type.u64, - &.{second_half}, - ); - try cg.emitWValue(intrin_ret_second); - if (bits == 128) { - try cg.store(.stack, .stack, Type.u64, result.offset() + 8); - try cg.store(.stack, .stack, Type.u64, result.offset()); - } else { - var tmp = try cg.allocLocal(Type.u64); - defer tmp.free(cg); - try cg.addLocal(.local_tee, tmp.local.value); - try cg.emitWValue(.{ .imm64 = 128 - bits }); - if (ty.isSignedInt(zcu)) { - try cg.addTag(.i64_shr_s); - } else { - try cg.addTag(.i64_shr_u); - } - try cg.store(.stack, .stack, Type.u64, result.offset() + 8); - try cg.addLocal(.local_get, tmp.local.value); - try cg.emitWValue(.{ .imm64 = bits - 64 }); - try cg.addTag(.i64_shl); - try cg.addTag(.i64_or); - try cg.store(.stack, .stack, Type.u64, result.offset()); - } - return cg.finishAir(inst, result, &.{ty_op.operand}); - }, - else => unreachable, - } -} - -fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - // Each entry to this table is a slice (ptr+len). - // The operand in this instruction represents the index within this table. - // This means to get the final name, we emit the base pointer and then perform - // pointer arithmetic to find the pointer to this slice and return that. - // - // As the names are global and the slice elements are constant, we do not have - // to make a copy of the ptr+value but can point towards them directly. - const pt = cg.pt; - const name_ty = Type.slice_const_u8_sentinel_0; - const abi_size = name_ty.abiSize(pt.zcu); - - // Lowers to a i32.const or i64.const with the error table memory address. - cg.error_name_table_ref_count += 1; - try cg.addTag(.error_name_table_ref); - try cg.emitWValue(operand); - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(@intCast(abi_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(abi_size); - try cg.addTag(.i64_mul); - try cg.addTag(.i64_add); - }, - } - - return cg.finishAir(inst, .stack, &.{un_op}); -} - -fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void { - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const slice_ptr = try cg.resolveInst(ty_op.operand); - const result = try cg.buildPointerOffset(slice_ptr, offset, .new); - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -/// NOTE: Allocates place for result on virtual stack, when integer size > 64 bits -fn intZeroValue(cg: *CodeGen, ty: Type) InnerError!WValue { - const zcu = cg.pt.zcu; - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: Implement intZeroValue for integer bitsize: {d}", .{int_info.bits}); - }; - switch (wasm_bits) { - 32 => return .{ .imm32 = 0 }, - 64 => return .{ .imm64 = 0 }, - 128 => { - const result = try cg.allocStack(ty); - try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0); - try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8); - return result; - }, - else => unreachable, - } -} - -fn airAddSubWithOverflow(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { - assert(op == .add or op == .sub); - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const lhs = try cg.resolveInst(extra.lhs); - const rhs = try cg.resolveInst(extra.rhs); - const ty = cg.typeOf(extra.lhs); - const pt = cg.pt; - const zcu = pt.zcu; - - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: Implement overflow arithmetic for vectors", .{}); - } - - const int_info = ty.intInfo(zcu); - const is_signed = int_info.signedness == .signed; - if (int_info.bits > 128) { - return cg.fail("TODO: Implement {{add/sub}}_with_overflow for integer bitsize: {d}", .{int_info.bits}); - } - - const op_result = try cg.wrapBinOp(lhs, rhs, ty, op); - var op_tmp = try op_result.toLocal(cg, ty); - defer op_tmp.free(cg); - - const cmp_op: std.math.CompareOperator = switch (op) { - .add => .lt, - .sub => .gt, - else => unreachable, - }; - const overflow_bit = if (is_signed) blk: { - const zero = try intZeroValue(cg, ty); - const rhs_is_neg = try cg.cmp(rhs, zero, ty, .lt); - const overflow_cmp = try cg.cmp(op_tmp, lhs, ty, cmp_op); - break :blk try cg.cmp(rhs_is_neg, overflow_cmp, Type.u1, .neq); - } else try cg.cmp(op_tmp, lhs, ty, cmp_op); - var bit_tmp = try overflow_bit.toLocal(cg, Type.u1); - defer bit_tmp.free(cg); - - const result = try cg.allocStack(cg.typeOfIndex(inst)); - const offset: u32 = @intCast(ty.abiSize(zcu)); - try cg.store(result, op_tmp, ty, 0); - try cg.store(result, bit_tmp, Type.u1, offset); - - return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); -} - -fn airShlWithOverflow(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const lhs = try cg.resolveInst(extra.lhs); - const rhs = try cg.resolveInst(extra.rhs); - const ty = cg.typeOf(extra.lhs); - const rhs_ty = cg.typeOf(extra.rhs); - - if (ty.isVector(zcu)) { - if (!rhs_ty.isVector(zcu)) { - return cg.fail("TODO: implement vector 'shl_with_overflow' with scalar rhs", .{}); - } else { - return cg.fail("TODO: implement vector 'shl_with_overflow'", .{}); - } - } - - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: implement 'shl_with_overflow' for integer bitsize: {d}", .{int_info.bits}); - }; - - // Ensure rhs is coerced to lhs as they must have the same WebAssembly types - // before we can perform any binary operation. - const rhs_wasm_bits = toWasmBits(rhs_ty.intInfo(zcu).bits).?; - // If wasm_bits == 128, compiler-rt expects i32 for shift - const rhs_final = if (wasm_bits != rhs_wasm_bits and wasm_bits == 64) blk: { - const rhs_casted = try cg.intcast(rhs, rhs_ty, ty); - break :blk try rhs_casted.toLocal(cg, ty); - } else rhs; - - var shl = try (try cg.wrapBinOp(lhs, rhs_final, ty, .shl)).toLocal(cg, ty); - defer shl.free(cg); - - const overflow_bit = blk: { - const shr = try cg.binOp(shl, rhs_final, ty, .shr); - break :blk try cg.cmp(shr, lhs, ty, .neq); - }; - var overflow_local = try overflow_bit.toLocal(cg, Type.u1); - defer overflow_local.free(cg); - - const result = try cg.allocStack(cg.typeOfIndex(inst)); - const offset: u32 = @intCast(ty.abiSize(zcu)); - try cg.store(result, shl, ty, 0); - try cg.store(result, overflow_local, Type.u1, offset); - - return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); -} - -fn airMulWithOverflow(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data; - - const lhs = try cg.resolveInst(extra.lhs); - const rhs = try cg.resolveInst(extra.rhs); - const ty = cg.typeOf(extra.lhs); - const pt = cg.pt; - const zcu = pt.zcu; - - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: Implement overflow arithmetic for vectors", .{}); - } - - // We store the bit if it's overflowed or not in this. As it's zero-initialized - // we only need to update it if an overflow (or underflow) occurred. - var overflow_bit = try cg.ensureAllocLocal(Type.u1); - defer overflow_bit.free(cg); - - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: Implement `@mulWithOverflow` for integer bitsize: {d}", .{int_info.bits}); - }; - - const zero: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = 0 }, - 64, 128 => .{ .imm64 = 0 }, - else => unreachable, - }; - - // for 32 bit integers we upcast it to a 64bit integer - const mul = if (wasm_bits == 32) blk: { - const new_ty = if (int_info.signedness == .signed) Type.i64 else Type.u64; - const lhs_upcast = try cg.intcast(lhs, ty, new_ty); - const rhs_upcast = try cg.intcast(rhs, ty, new_ty); - const bin_op = try (try cg.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(cg, new_ty); - const res = try (try cg.trunc(bin_op, ty, new_ty)).toLocal(cg, ty); - const res_upcast = try cg.intcast(res, ty, new_ty); - _ = try cg.cmp(res_upcast, bin_op, new_ty, .neq); - try cg.addLocal(.local_set, overflow_bit.local.value); - break :blk res; - } else if (wasm_bits == 64) blk: { - const new_ty = if (int_info.signedness == .signed) Type.i128 else Type.u128; - const lhs_upcast = try cg.intcast(lhs, ty, new_ty); - const rhs_upcast = try cg.intcast(rhs, ty, new_ty); - const bin_op = try (try cg.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(cg, new_ty); - const res = try (try cg.trunc(bin_op, ty, new_ty)).toLocal(cg, ty); - const res_upcast = try cg.intcast(res, ty, new_ty); - _ = try cg.cmp(res_upcast, bin_op, new_ty, .neq); - try cg.addLocal(.local_set, overflow_bit.local.value); - break :blk res; - } else if (int_info.bits == 128 and int_info.signedness == .unsigned) blk: { - var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64); - defer lhs_lsb.free(cg); - var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64); - defer lhs_msb.free(cg); - var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64); - defer rhs_lsb.free(cg); - var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64); - defer rhs_msb.free(cg); - - const cross_1 = try cg.callIntrinsic( - .__multi3, - &[_]InternPool.Index{.i64_type} ** 4, - Type.i128, - &.{ lhs_msb, zero, rhs_lsb, zero }, - ); - const cross_2 = try cg.callIntrinsic( - .__multi3, - &[_]InternPool.Index{.i64_type} ** 4, - Type.i128, - &.{ rhs_msb, zero, lhs_lsb, zero }, - ); - const mul_lsb = try cg.callIntrinsic( - .__multi3, - &[_]InternPool.Index{.i64_type} ** 4, - Type.i128, - &.{ rhs_lsb, zero, lhs_lsb, zero }, - ); - - const rhs_msb_not_zero = try cg.cmp(rhs_msb, zero, Type.u64, .neq); - const lhs_msb_not_zero = try cg.cmp(lhs_msb, zero, Type.u64, .neq); - const both_msb_not_zero = try cg.binOp(rhs_msb_not_zero, lhs_msb_not_zero, Type.bool, .@"and"); - const cross_1_msb = try cg.load(cross_1, Type.u64, 8); - const cross_1_msb_not_zero = try cg.cmp(cross_1_msb, zero, Type.u64, .neq); - const cond_1 = try cg.binOp(both_msb_not_zero, cross_1_msb_not_zero, Type.bool, .@"or"); - const cross_2_msb = try cg.load(cross_2, Type.u64, 8); - const cross_2_msb_not_zero = try cg.cmp(cross_2_msb, zero, Type.u64, .neq); - const cond_2 = try cg.binOp(cond_1, cross_2_msb_not_zero, Type.bool, .@"or"); - - const cross_1_lsb = try cg.load(cross_1, Type.u64, 0); - const cross_2_lsb = try cg.load(cross_2, Type.u64, 0); - const cross_add = try cg.binOp(cross_1_lsb, cross_2_lsb, Type.u64, .add); - - var mul_lsb_msb = try (try cg.load(mul_lsb, Type.u64, 8)).toLocal(cg, Type.u64); - defer mul_lsb_msb.free(cg); - var all_add = try (try cg.binOp(cross_add, mul_lsb_msb, Type.u64, .add)).toLocal(cg, Type.u64); - defer all_add.free(cg); - const add_overflow = try cg.cmp(all_add, mul_lsb_msb, Type.u64, .lt); - - // result for overflow bit - _ = try cg.binOp(cond_2, add_overflow, Type.bool, .@"or"); - try cg.addLocal(.local_set, overflow_bit.local.value); - - const tmp_result = try cg.allocStack(Type.u128); - try cg.emitWValue(tmp_result); - const mul_lsb_lsb = try cg.load(mul_lsb, Type.u64, 0); - try cg.store(.stack, mul_lsb_lsb, Type.u64, tmp_result.offset()); - try cg.store(tmp_result, all_add, Type.u64, 8); - break :blk tmp_result; - } else if (int_info.bits == 128 and int_info.signedness == .signed) blk: { - const overflow_ret = try cg.allocStack(Type.i32); - const res = try cg.callIntrinsic( - .__muloti4, - &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type }, - Type.i128, - &.{ lhs, rhs, overflow_ret }, - ); - _ = try cg.load(overflow_ret, Type.i32, 0); - try cg.addLocal(.local_set, overflow_bit.local.value); - break :blk res; - } else return cg.fail("TODO: @mulWithOverflow for {f}", .{ty.fmt(pt)}); - var bin_op_local = try mul.toLocal(cg, ty); - defer bin_op_local.free(cg); - - const result = try cg.allocStack(cg.typeOfIndex(inst)); - const offset: u32 = @intCast(ty.abiSize(zcu)); - try cg.store(result, bin_op_local, ty, 0); - try cg.store(result, overflow_bit, Type.u1, offset); - - return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); -} - -fn airMaxMin( - cg: *CodeGen, - inst: Air.Inst.Index, - op: enum { fmax, fmin }, - cmp_op: std.math.CompareOperator, -) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ty = cg.typeOfIndex(inst); - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: `@maximum` and `@minimum` for vectors", .{}); - } - - if (ty.abiSize(zcu) > 16) { - return cg.fail("TODO: `@maximum` and `@minimum` for types larger than 16 bytes", .{}); - } - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - if (ty.zigTypeTag(zcu) == .float) { - const intrinsic = switch (op) { - inline .fmin, .fmax => |ct_op| switch (ty.floatBits(cg.target)) { - inline 16, 32, 64, 80, 128 => |bits| @field( - Mir.Intrinsic, - libcFloatPrefix(bits) ++ @tagName(ct_op) ++ libcFloatSuffix(bits), - ), - else => unreachable, - }, - }; - const result = try cg.callIntrinsic(intrinsic, &.{ ty.ip_index, ty.ip_index }, ty, &.{ lhs, rhs }); - try cg.lowerToStack(result); - } else { - // operands to select from - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); - _ = try cg.cmp(lhs, rhs, ty, cmp_op); - - // based on the result from comparison, return operand 0 or 1. - try cg.addTag(.select); - } - - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airMulAdd(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data; - - const ty = cg.typeOfIndex(inst); - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: `@mulAdd` for vectors", .{}); - } - - const addend = try cg.resolveInst(pl_op.operand); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - const result = if (ty.floatBits(cg.target) == 16) fl_result: { - const rhs_ext = try cg.fpext(rhs, ty, Type.f32); - const lhs_ext = try cg.fpext(lhs, ty, Type.f32); - const addend_ext = try cg.fpext(addend, ty, Type.f32); - // call to compiler-rt `fn fmaf(f32, f32, f32) f32` - const result = try cg.callIntrinsic( - .fmaf, - &.{ .f32_type, .f32_type, .f32_type }, - Type.f32, - &.{ rhs_ext, lhs_ext, addend_ext }, - ); - break :fl_result try cg.fptrunc(result, Type.f32, ty); - } else result: { - const mul_result = try cg.binOp(lhs, rhs, ty, .mul); - break :result try cg.binOp(mul_result, addend, ty, .add); - }; - - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand }); -} - -fn airClz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const ty = cg.typeOf(ty_op.operand); - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: `@clz` for vectors", .{}); - } - - const operand = try cg.resolveInst(ty_op.operand); - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: `@clz` for integers with bitsize '{d}'", .{int_info.bits}); - }; - - switch (wasm_bits) { - 32 => { - if (int_info.signedness == .signed) { - const mask = ~@as(u32, 0) >> @intCast(32 - int_info.bits); - _ = try cg.binOp(operand, .{ .imm32 = mask }, ty, .@"and"); - } else { - try cg.emitWValue(operand); - } - try cg.addTag(.i32_clz); - }, - 64 => { - if (int_info.signedness == .signed) { - const mask = ~@as(u64, 0) >> @intCast(64 - int_info.bits); - _ = try cg.binOp(operand, .{ .imm64 = mask }, ty, .@"and"); - } else { - try cg.emitWValue(operand); - } - try cg.addTag(.i64_clz); - try cg.addTag(.i32_wrap_i64); - }, - 128 => { - var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64); - defer msb.free(cg); - - try cg.emitWValue(msb); - try cg.addTag(.i64_clz); - _ = try cg.load(operand, Type.u64, 0); - try cg.addTag(.i64_clz); - try cg.emitWValue(.{ .imm64 = 64 }); - try cg.addTag(.i64_add); - _ = try cg.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq); - try cg.addTag(.select); - try cg.addTag(.i32_wrap_i64); - }, - else => unreachable, - } - - if (wasm_bits != int_info.bits) { - try cg.emitWValue(.{ .imm32 = wasm_bits - int_info.bits }); - try cg.addTag(.i32_sub); - } - - return cg.finishAir(inst, .stack, &.{ty_op.operand}); -} - -fn airCtz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const ty = cg.typeOf(ty_op.operand); - - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: `@ctz` for vectors", .{}); - } - - const operand = try cg.resolveInst(ty_op.operand); - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: `@clz` for integers with bitsize '{d}'", .{int_info.bits}); - }; - - switch (wasm_bits) { - 32 => { - if (wasm_bits != int_info.bits) { - const val: u32 = @as(u32, 1) << @as(u5, @intCast(int_info.bits)); - // leave value on the stack - _ = try cg.binOp(operand, .{ .imm32 = val }, ty, .@"or"); - } else try cg.emitWValue(operand); - try cg.addTag(.i32_ctz); - }, - 64 => { - if (wasm_bits != int_info.bits) { - const val: u64 = @as(u64, 1) << @as(u6, @intCast(int_info.bits)); - // leave value on the stack - _ = try cg.binOp(operand, .{ .imm64 = val }, ty, .@"or"); - } else try cg.emitWValue(operand); - try cg.addTag(.i64_ctz); - try cg.addTag(.i32_wrap_i64); - }, - 128 => { - var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64); - defer lsb.free(cg); - - try cg.emitWValue(lsb); - try cg.addTag(.i64_ctz); - _ = try cg.load(operand, Type.u64, 8); - if (wasm_bits != int_info.bits) { - try cg.addImm64(@as(u64, 1) << @as(u6, @intCast(int_info.bits - 64))); - try cg.addTag(.i64_or); - } - try cg.addTag(.i64_ctz); - try cg.addImm64(64); - if (wasm_bits != int_info.bits) { - try cg.addTag(.i64_or); - } else { - try cg.addTag(.i64_add); - } - _ = try cg.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq); - try cg.addTag(.select); - try cg.addTag(.i32_wrap_i64); - }, - else => unreachable, - } - - return cg.finishAir(inst, .stack, &.{ty_op.operand}); -} - -fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const dbg_stmt = cg.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; - try cg.addInst(.{ .tag = .dbg_line, .data = .{ - .payload = try cg.addExtra(Mir.DbgLineColumn{ - .line = dbg_stmt.line, - .column = dbg_stmt.column, - }), - } }); - return cg.finishAir(inst, .none, &.{}); -} - -fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); - // TODO - try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); -} - -fn airDbgVar( - cg: *CodeGen, - inst: Air.Inst.Index, - local_tag: link.File.Dwarf.WipNav.LocalVarTag, - is_ptr: bool, -) InnerError!void { - _ = is_ptr; - _ = local_tag; - return cg.finishAir(inst, .none, &.{}); -} - -fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const err_union = try cg.resolveInst(pl_op.operand); - const extra = cg.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); - const err_union_ty = cg.typeOf(pl_op.operand); - const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false); - return cg.finishAir(inst, result, &.{pl_op.operand}); -} - -fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload); - const err_union_ptr = try cg.resolveInst(extra.data.ptr); - const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); - const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu); - const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true); - return cg.finishAir(inst, result, &.{extra.data.ptr}); -} - -fn lowerTry( - cg: *CodeGen, - inst: Air.Inst.Index, - err_union: WValue, - body: []const Air.Inst.Index, - err_union_ty: Type, - operand_is_ptr: bool, -) InnerError!WValue { - const zcu = cg.pt.zcu; - - const pl_ty = err_union_ty.errorUnionPayload(zcu); - const pl_has_bits = pl_ty.hasRuntimeBitsIgnoreComptime(zcu); - - if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { - // Block we can jump out of when error is not set - try cg.startBlock(.block, .empty); - - // check if the error tag is set for the error union. - try cg.emitWValue(err_union); - if (pl_has_bits or operand_is_ptr) { - const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); - try cg.addMemArg(.i32_load16_u, .{ - .offset = err_union.offset() + err_offset, - .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?), - }); - } - try cg.addTag(.i32_eqz); - try cg.addLabel(.br_if, 0); // jump out of block when error is '0' - - const liveness = cg.liveness.getCondBr(inst); - try cg.branches.append(cg.gpa, .{}); - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.else_deaths.len + liveness.then_deaths.len); - defer { - var branch = cg.branches.pop().?; - branch.deinit(cg.gpa); - } - try cg.genBody(body); - try cg.endBlock(); - } - - // if we reach here it means error was not set, and we want the payload - if (!pl_has_bits and !operand_is_ptr) { - return .none; - } - - const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); - if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) { - return buildPointerOffset(cg, err_union, pl_offset, .new); - } - const payload = try cg.load(err_union, pl_ty, pl_offset); - return payload.toLocal(cg, pl_ty); -} - -fn airByteSwap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const ty = cg.typeOfIndex(inst); - const operand = try cg.resolveInst(ty_op.operand); - - if (ty.zigTypeTag(zcu) == .vector) { - return cg.fail("TODO: @byteSwap for vectors", .{}); - } - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}); - }; - - // bytes are no-op - if (int_info.bits == 8) { - return cg.finishAir(inst, cg.reuseOperand(ty_op.operand, operand), &.{ty_op.operand}); - } - - const result = result: { - switch (wasm_bits) { - 32 => { - const intrin_ret = try cg.callIntrinsic( - .__bswapsi2, - &.{.u32_type}, - Type.u32, - &.{operand}, - ); - break :result if (int_info.bits == 32) - intrin_ret - else - try cg.binOp(intrin_ret, .{ .imm32 = 32 - int_info.bits }, ty, .shr); - }, - 64 => { - const intrin_ret = try cg.callIntrinsic( - .__bswapdi2, - &.{.u64_type}, - Type.u64, - &.{operand}, - ); - break :result if (int_info.bits == 64) - intrin_ret - else - try cg.binOp(intrin_ret, .{ .imm64 = 64 - int_info.bits }, ty, .shr); - }, - else => return cg.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}), - } - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -fn airDiv(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ty = cg.typeOfIndex(inst); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - const result = try cg.binOp(lhs, rhs, ty, .div); - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airDivTrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ty = cg.typeOfIndex(inst); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - const div_result = try cg.binOp(lhs, rhs, ty, .div); - - if (ty.isAnyFloat()) { - const trunc_result = try cg.floatOp(.trunc, ty, &.{div_result}); - return cg.finishAir(inst, trunc_result, &.{ bin_op.lhs, bin_op.rhs }); - } - - return cg.finishAir(inst, div_result, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airDivFloor(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const zcu = cg.pt.zcu; - const ty = cg.typeOfIndex(inst); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - if (ty.isUnsignedInt(zcu)) { - _ = try cg.binOp(lhs, rhs, ty, .div); - } else if (ty.isSignedInt(zcu)) { - const int_bits = ty.intInfo(zcu).bits; - const wasm_bits = toWasmBits(int_bits) orelse { - return cg.fail("TODO: `@divFloor` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); - }; - - if (wasm_bits > 64) { - return cg.fail("TODO: `@divFloor` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); - } - - const zero: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = 0 }, - 64 => .{ .imm64 = 0 }, - else => unreachable, - }; - - // tee leaves the value on the stack and stores it in a local. - const quotient = try cg.allocLocal(ty); - _ = try cg.binOp(lhs, rhs, ty, .div); - try cg.addLocal(.local_tee, quotient.local.value); - - // select takes a 32 bit value as the condition, so in the 64 bit case we use eqz to narrow - // the 64 bit value we want to use as the condition to 32 bits. - // This also inverts the condition (non 0 => 0, 0 => 1), so we put the adjusted and - // non-adjusted quotients on the stack in the opposite order for 32 vs 64 bits. - if (wasm_bits == 64) { - try cg.emitWValue(quotient); - } - - // 0 if the signs of rhs_wasm and lhs_wasm are the same, 1 otherwise. - _ = try cg.binOp(lhs, rhs, ty, .xor); - _ = try cg.cmp(.stack, zero, ty, .lt); - - switch (wasm_bits) { - 32 => { - try cg.addTag(.i32_sub); - try cg.emitWValue(quotient); - }, - 64 => { - try cg.addTag(.i64_extend_i32_u); - try cg.addTag(.i64_sub); - }, - else => unreachable, - } - - _ = try cg.binOp(lhs, rhs, ty, .rem); - - if (wasm_bits == 64) { - try cg.addTag(.i64_eqz); - } - - try cg.addTag(.select); - - // We need to zero the high bits because N bit comparisons consider all 32 or 64 bits, and - // expect all but the lowest N bits to be 0. - // TODO: Should we be zeroing the high bits here or should we be ignoring the high bits - // when performing comparisons? - if (int_bits != wasm_bits) { - _ = try cg.wrapOperand(.stack, ty); - } - } else { - const float_bits = ty.floatBits(cg.target); - if (float_bits > 64) { - return cg.fail("TODO: `@divFloor` for floats with bitsize: {d}", .{float_bits}); - } - const is_f16 = float_bits == 16; - - const lhs_wasm = if (is_f16) try cg.fpext(lhs, Type.f16, Type.f32) else lhs; - const rhs_wasm = if (is_f16) try cg.fpext(rhs, Type.f16, Type.f32) else rhs; - - try cg.emitWValue(lhs_wasm); - try cg.emitWValue(rhs_wasm); - - switch (float_bits) { - 16, 32 => { - try cg.addTag(.f32_div); - try cg.addTag(.f32_floor); - }, - 64 => { - try cg.addTag(.f64_div); - try cg.addTag(.f64_floor); - }, - else => unreachable, - } - - if (is_f16) { - _ = try cg.fptrunc(.stack, Type.f32, Type.f16); - } - } - - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airRem(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ty = cg.typeOfIndex(inst); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - const result = try cg.binOp(lhs, rhs, ty, .rem); - - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -/// Remainder after floor division, defined by: -/// @divFloor(a, b) * b + @mod(a, b) = a -fn airMod(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const pt = cg.pt; - const zcu = pt.zcu; - const ty = cg.typeOfIndex(inst); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - const result = result: { - if (ty.isUnsignedInt(zcu)) { - break :result try cg.binOp(lhs, rhs, ty, .rem); - } - if (ty.isSignedInt(zcu)) { - // The wasm rem instruction gives the remainder after truncating division (rounding towards - // 0), equivalent to @rem. - // We make use of the fact that: - // @mod(a, b) = @rem(@rem(a, b) + b, b) - const int_bits = ty.intInfo(zcu).bits; - const wasm_bits = toWasmBits(int_bits) orelse { - return cg.fail("TODO: `@mod` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); - }; - - if (wasm_bits > 64) { - return cg.fail("TODO: `@mod` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); - } - - _ = try cg.binOp(lhs, rhs, ty, .rem); - _ = try cg.binOp(.stack, rhs, ty, .add); - break :result try cg.binOp(.stack, rhs, ty, .rem); - } - if (ty.isAnyFloat()) { - const rem = try cg.binOp(lhs, rhs, ty, .rem); - const add = try cg.binOp(rem, rhs, ty, .add); - break :result try cg.binOp(add, rhs, ty, .rem); - } - return cg.fail("TODO: @mod for {f}", .{ty.fmt(pt)}); - }; - - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airSatMul(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const pt = cg.pt; - const zcu = pt.zcu; - const ty = cg.typeOfIndex(inst); - const int_info = ty.intInfo(zcu); - const is_signed = int_info.signedness == .signed; - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const wasm_bits = toWasmBits(int_info.bits) orelse { - return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)}); - }; - - switch (wasm_bits) { - 32 => { - const upcast_ty: Type = if (is_signed) Type.i64 else Type.u64; - const lhs_up = try cg.intcast(lhs, ty, upcast_ty); - const rhs_up = try cg.intcast(rhs, ty, upcast_ty); - var mul_res = try (try cg.binOp(lhs_up, rhs_up, upcast_ty, .mul)).toLocal(cg, upcast_ty); - defer mul_res.free(cg); - if (is_signed) { - const imm_max: WValue = .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - (int_info.bits - 1)) }; - try cg.emitWValue(mul_res); - try cg.emitWValue(imm_max); - _ = try cg.cmp(mul_res, imm_max, upcast_ty, .lt); - try cg.addTag(.select); - - var tmp = try cg.allocLocal(upcast_ty); - defer tmp.free(cg); - try cg.addLocal(.local_set, tmp.local.value); - - const imm_min: WValue = .{ .imm64 = ~@as(u64, 0) << @intCast(int_info.bits - 1) }; - try cg.emitWValue(tmp); - try cg.emitWValue(imm_min); - _ = try cg.cmp(tmp, imm_min, upcast_ty, .gt); - try cg.addTag(.select); - } else { - const imm_max: WValue = .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_info.bits) }; - try cg.emitWValue(mul_res); - try cg.emitWValue(imm_max); - _ = try cg.cmp(mul_res, imm_max, upcast_ty, .lt); - try cg.addTag(.select); - } - try cg.addTag(.i32_wrap_i64); - }, - 64 => { - if (!(int_info.bits == 64 and int_info.signedness == .signed)) { - return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)}); - } - const overflow_ret = try cg.allocStack(Type.i32); - _ = try cg.callIntrinsic( - .__mulodi4, - &[_]InternPool.Index{ .i64_type, .i64_type, .usize_type }, - Type.i64, - &.{ lhs, rhs, overflow_ret }, - ); - const xor = try cg.binOp(lhs, rhs, Type.i64, .xor); - const sign_v = try cg.binOp(xor, .{ .imm64 = 63 }, Type.i64, .shr); - _ = try cg.binOp(sign_v, .{ .imm64 = ~@as(u63, 0) }, Type.i64, .xor); - _ = try cg.load(overflow_ret, Type.i32, 0); - try cg.addTag(.i32_eqz); - try cg.addTag(.select); - }, - 128 => { - if (!(int_info.bits == 128 and int_info.signedness == .signed)) { - return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)}); - } - const overflow_ret = try cg.allocStack(Type.i32); - const ret = try cg.callIntrinsic( - .__muloti4, - &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type }, - Type.i128, - &.{ lhs, rhs, overflow_ret }, - ); - try cg.lowerToStack(ret); - const xor = try cg.binOp(lhs, rhs, Type.i128, .xor); - const sign_v = try cg.binOp(xor, .{ .imm32 = 127 }, Type.i128, .shr); - - // xor ~@as(u127, 0) - try cg.emitWValue(sign_v); - const lsb = try cg.load(sign_v, Type.u64, 0); - _ = try cg.binOp(lsb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor); - try cg.store(.stack, .stack, Type.u64, sign_v.offset()); - try cg.emitWValue(sign_v); - const msb = try cg.load(sign_v, Type.u64, 8); - _ = try cg.binOp(msb, .{ .imm64 = ~@as(u63, 0) }, Type.u64, .xor); - try cg.store(.stack, .stack, Type.u64, sign_v.offset() + 8); - - try cg.lowerToStack(sign_v); - _ = try cg.load(overflow_ret, Type.i32, 0); - try cg.addTag(.i32_eqz); - try cg.addTag(.select); - }, - else => unreachable, - } - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airSatBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { - assert(op == .add or op == .sub); - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const zcu = cg.pt.zcu; - const ty = cg.typeOfIndex(inst); - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - - const int_info = ty.intInfo(zcu); - const is_signed = int_info.signedness == .signed; - - if (int_info.bits > 64) { - return cg.fail("TODO: saturating arithmetic for integers with bitsize '{d}'", .{int_info.bits}); - } - - if (is_signed) { - const result = try signedSat(cg, lhs, rhs, ty, op); - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); - } - - const wasm_bits = toWasmBits(int_info.bits).?; - var bin_result = try (try cg.binOp(lhs, rhs, ty, op)).toLocal(cg, ty); - defer bin_result.free(cg); - if (wasm_bits != int_info.bits and op == .add) { - const val: u64 = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(int_info.bits))) - 1)); - const imm_val: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = @intCast(val) }, - 64 => .{ .imm64 = val }, - else => unreachable, - }; - - try cg.emitWValue(bin_result); - try cg.emitWValue(imm_val); - _ = try cg.cmp(bin_result, imm_val, ty, .lt); - } else { - switch (wasm_bits) { - 32 => try cg.addImm32(if (op == .add) std.math.maxInt(u32) else 0), - 64 => try cg.addImm64(if (op == .add) std.math.maxInt(u64) else 0), - else => unreachable, - } - try cg.emitWValue(bin_result); - _ = try cg.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt); - } - - try cg.addTag(.select); - return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn signedSat(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { - const pt = cg.pt; - const zcu = pt.zcu; - const int_info = ty.intInfo(zcu); - const wasm_bits = toWasmBits(int_info.bits).?; - const is_wasm_bits = wasm_bits == int_info.bits; - const ext_ty = if (!is_wasm_bits) try pt.intType(int_info.signedness, wasm_bits) else ty; - - const max_val: u64 = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(int_info.bits - 1))) - 1)); - const min_val: i64 = (-@as(i64, @intCast(@as(u63, @intCast(max_val))))) - 1; - const max_wvalue: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = @truncate(max_val) }, - 64 => .{ .imm64 = max_val }, - else => unreachable, - }; - const min_wvalue: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = @bitCast(@as(i32, @truncate(min_val))) }, - 64 => .{ .imm64 = @bitCast(min_val) }, - else => unreachable, - }; - - var bin_result = try (try cg.binOp(lhs, rhs, ext_ty, op)).toLocal(cg, ext_ty); - if (!is_wasm_bits) { - defer bin_result.free(cg); // not returned in this branch - try cg.emitWValue(bin_result); - try cg.emitWValue(max_wvalue); - _ = try cg.cmp(bin_result, max_wvalue, ext_ty, .lt); - try cg.addTag(.select); - try cg.addLocal(.local_set, bin_result.local.value); // re-use local - - try cg.emitWValue(bin_result); - try cg.emitWValue(min_wvalue); - _ = try cg.cmp(bin_result, min_wvalue, ext_ty, .gt); - try cg.addTag(.select); - try cg.addLocal(.local_set, bin_result.local.value); // re-use local - return (try cg.wrapOperand(bin_result, ty)).toLocal(cg, ty); - } else { - const zero: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = 0 }, - 64 => .{ .imm64 = 0 }, - else => unreachable, - }; - try cg.emitWValue(max_wvalue); - try cg.emitWValue(min_wvalue); - _ = try cg.cmp(bin_result, zero, ty, .lt); - try cg.addTag(.select); - try cg.emitWValue(bin_result); - // leave on stack - const cmp_zero_result = try cg.cmp(rhs, zero, ty, if (op == .add) .lt else .gt); - const cmp_bin_result = try cg.cmp(bin_result, lhs, ty, .lt); - _ = try cg.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor. - try cg.addTag(.select); - try cg.addLocal(.local_set, bin_result.local.value); // re-use local - return bin_result; - } -} - -fn airShlSat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const pt = cg.pt; - const zcu = pt.zcu; - - if (cg.typeOf(bin_op.lhs).isVector(zcu) and !cg.typeOf(bin_op.rhs).isVector(zcu)) { - return cg.fail("TODO: implement vector 'shl_sat' with scalar rhs", .{}); - } - - const ty = cg.typeOfIndex(inst); - const int_info = ty.intInfo(zcu); - const is_signed = int_info.signedness == .signed; - if (int_info.bits > 64) { - return cg.fail("TODO: Saturating shifting left for integers with bitsize '{d}'", .{int_info.bits}); - } - - const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); - const wasm_bits = toWasmBits(int_info.bits).?; - const result = try cg.allocLocal(ty); - - if (wasm_bits == int_info.bits) { - var shl = try (try cg.binOp(lhs, rhs, ty, .shl)).toLocal(cg, ty); - defer shl.free(cg); - var shr = try (try cg.binOp(shl, rhs, ty, .shr)).toLocal(cg, ty); - defer shr.free(cg); - - switch (wasm_bits) { - 32 => blk: { - if (!is_signed) { - try cg.addImm32(std.math.maxInt(u32)); - break :blk; - } - try cg.addImm32(@bitCast(@as(i32, std.math.minInt(i32)))); - try cg.addImm32(@bitCast(@as(i32, std.math.maxInt(i32)))); - _ = try cg.cmp(lhs, .{ .imm32 = 0 }, ty, .lt); - try cg.addTag(.select); - }, - 64 => blk: { - if (!is_signed) { - try cg.addImm64(std.math.maxInt(u64)); - break :blk; - } - try cg.addImm64(@bitCast(@as(i64, std.math.minInt(i64)))); - try cg.addImm64(@bitCast(@as(i64, std.math.maxInt(i64)))); - _ = try cg.cmp(lhs, .{ .imm64 = 0 }, ty, .lt); - try cg.addTag(.select); - }, - else => unreachable, - } - try cg.emitWValue(shl); - _ = try cg.cmp(lhs, shr, ty, .neq); - try cg.addTag(.select); - try cg.addLocal(.local_set, result.local.value); - } else { - const shift_size = wasm_bits - int_info.bits; - const shift_value: WValue = switch (wasm_bits) { - 32 => .{ .imm32 = shift_size }, - 64 => .{ .imm64 = shift_size }, - else => unreachable, - }; - const ext_ty = try pt.intType(int_info.signedness, wasm_bits); - - var shl_res = try (try cg.binOp(lhs, shift_value, ext_ty, .shl)).toLocal(cg, ext_ty); - defer shl_res.free(cg); - var shl = try (try cg.binOp(shl_res, rhs, ext_ty, .shl)).toLocal(cg, ext_ty); - defer shl.free(cg); - var shr = try (try cg.binOp(shl, rhs, ext_ty, .shr)).toLocal(cg, ext_ty); - defer shr.free(cg); - - switch (wasm_bits) { - 32 => blk: { - if (!is_signed) { - try cg.addImm32(std.math.maxInt(u32)); - break :blk; - } - - try cg.addImm32(@bitCast(@as(i32, std.math.minInt(i32)))); - try cg.addImm32(@bitCast(@as(i32, std.math.maxInt(i32)))); - _ = try cg.cmp(shl_res, .{ .imm32 = 0 }, ext_ty, .lt); - try cg.addTag(.select); - }, - 64 => blk: { - if (!is_signed) { - try cg.addImm64(std.math.maxInt(u64)); - break :blk; - } - - try cg.addImm64(@bitCast(@as(i64, std.math.minInt(i64)))); - try cg.addImm64(@bitCast(@as(i64, std.math.maxInt(i64)))); - _ = try cg.cmp(shl_res, .{ .imm64 = 0 }, ext_ty, .lt); - try cg.addTag(.select); - }, - else => unreachable, - } - try cg.emitWValue(shl); - _ = try cg.cmp(shl_res, shr, ext_ty, .neq); - try cg.addTag(.select); - try cg.addLocal(.local_set, result.local.value); - var shift_result = try cg.binOp(result, shift_value, ext_ty, .shr); - if (is_signed) { - shift_result = try cg.wrapOperand(shift_result, ty); - } - try cg.addLocal(.local_set, result.local.value); - } - - return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); -} - -/// Calls a compiler-rt intrinsic by creating an undefined symbol, -/// then lowering the arguments and calling the symbol as a function call. -/// This function call assumes the C-ABI. -/// Asserts arguments are not stack values when the return value is -/// passed as the first parameter. -/// May leave the return value on the stack. -fn callIntrinsic( - cg: *CodeGen, - intrinsic: Mir.Intrinsic, - param_types: []const InternPool.Index, - return_type: Type, - args: []const WValue, -) InnerError!WValue { - assert(param_types.len == args.len); - const zcu = cg.pt.zcu; - - // Always pass over C-ABI - - const want_sret_param = firstParamSRet(.{ .wasm_mvp = .{} }, return_type, zcu, cg.target); - // if we want return as first param, we allocate a pointer to stack, - // and emit it as our first argument - const sret = if (want_sret_param) blk: { - const sret_local = try cg.allocStack(return_type); - try cg.lowerToStack(sret_local); - break :blk sret_local; - } else .none; - - // Lower all arguments to the stack before we call our function - for (args, 0..) |arg, arg_i| { - assert(!(want_sret_param and arg == .stack)); - assert(Type.fromInterned(param_types[arg_i]).hasRuntimeBitsIgnoreComptime(zcu)); - try cg.lowerArg(.{ .wasm_mvp = .{} }, Type.fromInterned(param_types[arg_i]), arg); - } - - try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } }); - - if (!return_type.hasRuntimeBitsIgnoreComptime(zcu)) { - return .none; - } else if (return_type.isNoReturn(zcu)) { - try cg.addTag(.@"unreachable"); - return .none; - } else if (want_sret_param) { - return sret; - } else { - return .stack; - } -} - -fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try cg.resolveInst(un_op); - const enum_ty = cg.typeOf(un_op); - - const result_ptr = try cg.allocStack(cg.typeOfIndex(inst)); - try cg.lowerToStack(result_ptr); - try cg.emitWValue(operand); - try cg.addInst(.{ .tag = .call_tag_name, .data = .{ .ip_index = enum_ty.toIntern() } }); - - return cg.finishAir(inst, result_ptr, &.{un_op}); -} - -fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try cg.resolveInst(ty_op.operand); - const error_set_ty = ty_op.ty.toType(); - const result = try cg.allocLocal(Type.bool); - - const names = error_set_ty.errorSetNames(zcu); - var values = try std.array_list.Managed(u32).initCapacity(cg.gpa, names.len); - defer values.deinit(); - - var lowest: ?u32 = null; - var highest: ?u32 = null; - for (0..names.len) |name_index| { - const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?; - if (lowest) |*l| { - if (err_int < l.*) { - l.* = err_int; - } - } else { - lowest = err_int; - } - if (highest) |*h| { - if (err_int > h.*) { - highest = err_int; - } - } else { - highest = err_int; - } - - values.appendAssumeCapacity(err_int); - } - - // start block for 'true' branch - try cg.startBlock(.block, .empty); - // start block for 'false' branch - try cg.startBlock(.block, .empty); - // block for the jump table itself - try cg.startBlock(.block, .empty); - - // lower operand to determine jump table target - try cg.emitWValue(operand); - try cg.addImm32(lowest.?); - try cg.addTag(.i32_sub); - - // Account for default branch so always add '1' - const depth = @as(u32, @intCast(highest.? - lowest.? + 1)); - const jump_table: Mir.JumpTable = .{ .length = depth }; - const table_extra_index = try cg.addExtra(jump_table); - try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } }); - try cg.mir_extra.ensureUnusedCapacity(cg.gpa, depth); - - var value: u32 = lowest.?; - while (value <= highest.?) : (value += 1) { - const idx: u32 = blk: { - for (values.items) |val| { - if (val == value) break :blk 1; - } - break :blk 0; - }; - cg.mir_extra.appendAssumeCapacity(idx); - } - try cg.endBlock(); - - // 'false' branch (i.e. error set does not have value - // ensure we set local to 0 in case the local was re-used. - try cg.addImm32(0); - try cg.addLocal(.local_set, result.local.value); - try cg.addLabel(.br, 1); - try cg.endBlock(); - - // 'true' branch - try cg.addImm32(1); - try cg.addLocal(.local_set, result.local.value); - try cg.addLabel(.br, 0); - try cg.endBlock(); - - return cg.finishAir(inst, result, &.{ty_op.operand}); -} - -inline fn useAtomicFeature(cg: *const CodeGen) bool { - return cg.target.cpu.has(.wasm, .atomics); -} - -fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data; - - const ptr_ty = cg.typeOf(extra.ptr); - const ty = ptr_ty.childType(zcu); - const result_ty = cg.typeOfIndex(inst); - - const ptr_operand = try cg.resolveInst(extra.ptr); - const expected_val = try cg.resolveInst(extra.expected_value); - const new_val = try cg.resolveInst(extra.new_value); - - const cmp_result = try cg.allocLocal(Type.bool); - - const ptr_val = if (cg.useAtomicFeature()) val: { - const val_local = try cg.allocLocal(ty); - try cg.emitWValue(ptr_operand); - try cg.lowerToStack(expected_val); - try cg.lowerToStack(new_val); - try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) { - 1 => .i32_atomic_rmw8_cmpxchg_u, - 2 => .i32_atomic_rmw16_cmpxchg_u, - 4 => .i32_atomic_rmw_cmpxchg, - 8 => .i32_atomic_rmw_cmpxchg, - else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), - }, .{ - .offset = ptr_operand.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }); - try cg.addLocal(.local_tee, val_local.local.value); - _ = try cg.cmp(.stack, expected_val, ty, .eq); - try cg.addLocal(.local_set, cmp_result.local.value); - break :val val_local; - } else val: { - if (ty.abiSize(zcu) > 8) { - return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{}); - } - const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty); - - try cg.lowerToStack(ptr_operand); - try cg.lowerToStack(new_val); - try cg.emitWValue(ptr_val); - _ = try cg.cmp(ptr_val, expected_val, ty, .eq); - try cg.addLocal(.local_tee, cmp_result.local.value); - try cg.addTag(.select); - try cg.store(.stack, .stack, ty, 0); - - break :val ptr_val; - }; - - const result = if (isByRef(result_ty, zcu, cg.target)) val: { - try cg.emitWValue(cmp_result); - try cg.addImm32(~@as(u32, 0)); - try cg.addTag(.i32_xor); - try cg.addImm32(1); - try cg.addTag(.i32_and); - const and_result = try WValue.toLocal(.stack, cg, Type.bool); - const result_ptr = try cg.allocStack(result_ty); - try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu)))); - try cg.store(result_ptr, ptr_val, ty, 0); - break :val result_ptr; - } else val: { - try cg.addImm32(0); - try cg.emitWValue(ptr_val); - try cg.emitWValue(cmp_result); - try cg.addTag(.select); - break :val .stack; - }; - - return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value }); -} - -fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const atomic_load = cg.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; - const ptr = try cg.resolveInst(atomic_load.ptr); - const ty = cg.typeOfIndex(inst); - - if (cg.useAtomicFeature()) { - const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { - 1 => .i32_atomic_load8_u, - 2 => .i32_atomic_load16_u, - 4 => .i32_atomic_load, - 8 => .i64_atomic_load, - else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), - }; - try cg.emitWValue(ptr); - try cg.addAtomicMemArg(tag, .{ - .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }); - } else { - _ = try cg.load(ptr, ty, 0); - } - - return cg.finishAir(inst, .stack, &.{atomic_load.ptr}); -} - -fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = cg.air.extraData(Air.AtomicRmw, pl_op.payload).data; - - const ptr = try cg.resolveInst(pl_op.operand); - const operand = try cg.resolveInst(extra.operand); - const ty = cg.typeOfIndex(inst); - const op: std.builtin.AtomicRmwOp = extra.op(); - - if (cg.useAtomicFeature()) { - switch (op) { - .Max, - .Min, - .Nand, - => { - const tmp = try cg.load(ptr, ty, 0); - const value = try tmp.toLocal(cg, ty); - - // create a loop to cmpxchg the new value - try cg.startBlock(.loop, .empty); - - try cg.emitWValue(ptr); - try cg.emitWValue(value); - if (op == .Nand) { - const wasm_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?; - - const and_res = try cg.binOp(value, operand, ty, .@"and"); - if (wasm_bits == 32) - try cg.addImm32(~@as(u32, 0)) - else if (wasm_bits == 64) - try cg.addImm64(~@as(u64, 0)) - else - return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); - _ = try cg.binOp(and_res, .stack, ty, .xor); - } else { - try cg.emitWValue(value); - try cg.emitWValue(operand); - _ = try cg.cmp(value, operand, ty, if (op == .Max) .gt else .lt); - try cg.addTag(.select); - } - try cg.addAtomicMemArg( - switch (ty.abiSize(zcu)) { - 1 => .i32_atomic_rmw8_cmpxchg_u, - 2 => .i32_atomic_rmw16_cmpxchg_u, - 4 => .i32_atomic_rmw_cmpxchg, - 8 => .i64_atomic_rmw_cmpxchg, - else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}), - }, - .{ - .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }, - ); - const select_res = try cg.allocLocal(ty); - try cg.addLocal(.local_tee, select_res.local.value); - _ = try cg.cmp(.stack, value, ty, .neq); // leave on stack so we can use it for br_if - - try cg.emitWValue(select_res); - try cg.addLocal(.local_set, value.local.value); - - try cg.addLabel(.br_if, 0); - try cg.endBlock(); - return cg.finishAir(inst, value, &.{ pl_op.operand, extra.operand }); - }, - - // the other operations have their own instructions for Wasm. - else => { - try cg.emitWValue(ptr); - try cg.emitWValue(operand); - const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { - 1 => switch (op) { - .Xchg => .i32_atomic_rmw8_xchg_u, - .Add => .i32_atomic_rmw8_add_u, - .Sub => .i32_atomic_rmw8_sub_u, - .And => .i32_atomic_rmw8_and_u, - .Or => .i32_atomic_rmw8_or_u, - .Xor => .i32_atomic_rmw8_xor_u, - else => unreachable, - }, - 2 => switch (op) { - .Xchg => .i32_atomic_rmw16_xchg_u, - .Add => .i32_atomic_rmw16_add_u, - .Sub => .i32_atomic_rmw16_sub_u, - .And => .i32_atomic_rmw16_and_u, - .Or => .i32_atomic_rmw16_or_u, - .Xor => .i32_atomic_rmw16_xor_u, - else => unreachable, - }, - 4 => switch (op) { - .Xchg => .i32_atomic_rmw_xchg, - .Add => .i32_atomic_rmw_add, - .Sub => .i32_atomic_rmw_sub, - .And => .i32_atomic_rmw_and, - .Or => .i32_atomic_rmw_or, - .Xor => .i32_atomic_rmw_xor, - else => unreachable, - }, - 8 => switch (op) { - .Xchg => .i64_atomic_rmw_xchg, - .Add => .i64_atomic_rmw_add, - .Sub => .i64_atomic_rmw_sub, - .And => .i64_atomic_rmw_and, - .Or => .i64_atomic_rmw_or, - .Xor => .i64_atomic_rmw_xor, - else => unreachable, - }, - else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}), - }; - try cg.addAtomicMemArg(tag, .{ - .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }); - return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand }); - }, - } - } else { - const loaded = try cg.load(ptr, ty, 0); - const result = try loaded.toLocal(cg, ty); - - switch (op) { - .Xchg => { - try cg.store(ptr, operand, ty, 0); - }, - .Add, - .Sub, - .And, - .Or, - .Xor, - => { - try cg.emitWValue(ptr); - _ = try cg.binOp(result, operand, ty, switch (op) { - .Add => .add, - .Sub => .sub, - .And => .@"and", - .Or => .@"or", - .Xor => .xor, - else => unreachable, - }); - if (ty.isInt(zcu) and (op == .Add or op == .Sub)) { - _ = try cg.wrapOperand(.stack, ty); - } - try cg.store(.stack, .stack, ty, ptr.offset()); - }, - .Max, - .Min, - => { - try cg.emitWValue(ptr); - try cg.emitWValue(result); - try cg.emitWValue(operand); - _ = try cg.cmp(result, operand, ty, if (op == .Max) .gt else .lt); - try cg.addTag(.select); - try cg.store(.stack, .stack, ty, ptr.offset()); - }, - .Nand => { - const wasm_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?; - - try cg.emitWValue(ptr); - const and_res = try cg.binOp(result, operand, ty, .@"and"); - if (wasm_bits == 32) - try cg.addImm32(~@as(u32, 0)) - else if (wasm_bits == 64) - try cg.addImm64(~@as(u64, 0)) - else - return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); - _ = try cg.binOp(and_res, .stack, ty, .xor); - try cg.store(.stack, .stack, ty, ptr.offset()); - }, - } - - return cg.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); - } -} - -fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const zcu = cg.pt.zcu; - const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ptr = try cg.resolveInst(bin_op.lhs); - const operand = try cg.resolveInst(bin_op.rhs); - const ptr_ty = cg.typeOf(bin_op.lhs); - const ty = ptr_ty.childType(zcu); - - if (cg.useAtomicFeature()) { - const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { - 1 => .i32_atomic_store8, - 2 => .i32_atomic_store16, - 4 => .i32_atomic_store, - 8 => .i64_atomic_store, - else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), - }; - try cg.emitWValue(ptr); - try cg.lowerToStack(operand); - try cg.addAtomicMemArg(tag, .{ - .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), - }); - } else { - try cg.store(ptr, operand, ty, 0); - } - - return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); -} - -fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - try cg.emitWValue(cg.bottom_stack_value); - return cg.finishAir(inst, .stack, &.{}); -} - -fn airRuntimeNavPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - const ty_nav = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; - const mod = cg.pt.zcu.navFileScope(cg.owner_nav).mod.?; - if (mod.single_threaded) { - const result: WValue = .{ .nav_ref = .{ - .nav_index = ty_nav.nav, - .offset = 0, - } }; - return cg.finishAir(inst, result, &.{}); - } - return cg.fail("TODO: thread-local variables", .{}); -} - -fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type { - const zcu = cg.pt.zcu; - return cg.air.typeOf(inst, &zcu.intern_pool); -} - -fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type { - const zcu = cg.pt.zcu; - return cg.air.typeOfIndex(inst, &zcu.intern_pool); -} - -fn floatCmpIntrinsic(op: std.math.CompareOperator, bits: u16) Mir.Intrinsic { - return switch (op) { - .lt => switch (bits) { - 80 => .__ltxf2, - 128 => .__lttf2, - else => unreachable, - }, - .lte => switch (bits) { - 80 => .__lexf2, - 128 => .__letf2, - else => unreachable, - }, - .eq => switch (bits) { - 80 => .__eqxf2, - 128 => .__eqtf2, - else => unreachable, - }, - .neq => switch (bits) { - 80 => .__nexf2, - 128 => .__netf2, - else => unreachable, - }, - .gte => switch (bits) { - 80 => .__gexf2, - 128 => .__getf2, - else => unreachable, - }, - .gt => switch (bits) { - 80 => .__gtxf2, - 128 => .__gttf2, - else => unreachable, - }, - }; -} diff --git a/src/arch/wasm/Emit.zig b/src/arch/wasm/Emit.zig deleted file mode 100644 index 272d5519a6ff6c06f87a601fb58063e9fa3a5077..0000000000000000000000000000000000000000 --- a/src/arch/wasm/Emit.zig +++ /dev/null @@ -1,996 +0,0 @@ -const Emit = @This(); - -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; - -const Wasm = link.File.Wasm; -const Mir = @import("Mir.zig"); -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); -const InternPool = @import("../../InternPool.zig"); -const codegen = @import("../../codegen.zig"); - -mir: Mir, -wasm: *Wasm, -/// The binary representation that will be emitted by this module. -code: *ArrayList(u8), - -pub const Error = error{ - OutOfMemory, -}; - -pub fn lowerToCode(emit: *Emit) Error!void { - const mir = &emit.mir; - const code = emit.code; - const wasm = emit.wasm; - const comp = wasm.base.comp; - const gpa = comp.gpa; - const is_obj = comp.config.output_mode == .Obj; - const target = &comp.root_mod.resolved_target.result; - const is_wasm32 = target.cpu.arch == .wasm32; - - const tags = mir.instructions.items(.tag); - const datas = mir.instructions.items(.data); - var inst: u32 = 0; - - loop: switch (tags[inst]) { - .dbg_epilogue_begin => { - return; - }, - .block, .loop => { - const block_type = datas[inst].block_type; - try code.ensureUnusedCapacity(gpa, 2); - code.appendAssumeCapacity(@intFromEnum(tags[inst])); - code.appendAssumeCapacity(@intFromEnum(block_type)); - - inst += 1; - continue :loop tags[inst]; - }, - .uav_ref => { - if (is_obj) { - try uavRefObj(wasm, code, datas[inst].ip_index, 0, is_wasm32); - } else { - try uavRefExe(wasm, code, datas[inst].ip_index, 0, is_wasm32); - } - inst += 1; - continue :loop tags[inst]; - }, - .uav_ref_off => { - const extra = mir.extraData(Mir.UavRefOff, datas[inst].payload).data; - if (is_obj) { - try uavRefObj(wasm, code, extra.value, extra.offset, is_wasm32); - } else { - try uavRefExe(wasm, code, extra.value, extra.offset, is_wasm32); - } - inst += 1; - continue :loop tags[inst]; - }, - .nav_ref => { - try navRefOff(wasm, code, .{ .nav_index = datas[inst].nav_index, .offset = 0 }, is_wasm32); - inst += 1; - continue :loop tags[inst]; - }, - .nav_ref_off => { - try navRefOff(wasm, code, mir.extraData(Mir.NavRefOff, datas[inst].payload).data, is_wasm32); - inst += 1; - continue :loop tags[inst]; - }, - .func_ref => { - const indirect_func_idx: Wasm.ZcuIndirectFunctionSetIndex = @enumFromInt( - wasm.zcu_indirect_function_set.getIndex(datas[inst].nav_index).?, - ); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i32_const)); - if (is_obj) { - @panic("TODO"); - } else { - writeUleb128(code, 1 + @intFromEnum(indirect_func_idx)); - } - inst += 1; - continue :loop tags[inst]; - }, - .dbg_line => { - inst += 1; - continue :loop tags[inst]; - }, - .errors_len => { - try code.ensureUnusedCapacity(gpa, 6); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i32_const)); - // MIR is lowered during flush, so there is indeed only one thread at this time. - const errors_len = 1 + comp.zcu.?.intern_pool.global_error_set.getNamesFromMainThread().len; - writeSleb128(code, errors_len); - - inst += 1; - continue :loop tags[inst]; - }, - .error_name_table_ref => { - wasm.error_name_table_ref_count += 1; - try code.ensureUnusedCapacity(gpa, 11); - const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; - code.appendAssumeCapacity(@intFromEnum(opcode)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.errorNameTableSymbolIndex() }, - .tag = if (is_wasm32) .memory_addr_leb else .memory_addr_leb64, - .addend = 0, - }); - code.appendNTimesAssumeCapacity(0, if (is_wasm32) 5 else 10); - - inst += 1; - continue :loop tags[inst]; - } else { - const addr: u32 = wasm.errorNameTableAddr(); - writeSleb128(code, addr); - - inst += 1; - continue :loop tags[inst]; - } - }, - .br_if, .br, .memory_grow, .memory_size => { - try code.ensureUnusedCapacity(gpa, 11); - code.appendAssumeCapacity(@intFromEnum(tags[inst])); - writeUleb128(code, datas[inst].label); - - inst += 1; - continue :loop tags[inst]; - }, - - .local_get, .local_set, .local_tee => { - try code.ensureUnusedCapacity(gpa, 11); - code.appendAssumeCapacity(@intFromEnum(tags[inst])); - writeUleb128(code, datas[inst].local); - - inst += 1; - continue :loop tags[inst]; - }, - - .br_table => { - const extra_index = datas[inst].payload; - const extra = mir.extraData(Mir.JumpTable, extra_index); - const labels = mir.extra[extra.end..][0..extra.data.length]; - try code.ensureUnusedCapacity(gpa, 11 + 10 * labels.len); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.br_table)); - // -1 because default label is not part of length/depth. - writeUleb128(code, extra.data.length - 1); - for (labels) |label| writeUleb128(code, label); - - inst += 1; - continue :loop tags[inst]; - }, - - .call_nav => { - try code.ensureUnusedCapacity(gpa, 6); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.navSymbolIndex(datas[inst].nav_index) }, - .tag = .function_index_leb, - .addend = 0, - }); - code.appendNTimesAssumeCapacity(0, 5); - } else { - appendOutputFunctionIndex(code, .fromIpNav(wasm, datas[inst].nav_index)); - } - - inst += 1; - continue :loop tags[inst]; - }, - - .call_indirect => { - try code.ensureUnusedCapacity(gpa, 11); - const fn_info = comp.zcu.?.typeToFunc(.fromInterned(datas[inst].ip_index)).?; - const func_ty_index = wasm.getExistingFunctionType( - fn_info.cc, - fn_info.param_types.get(&comp.zcu.?.intern_pool), - .fromInterned(fn_info.return_type), - target, - ).?; - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call_indirect)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .type_index = func_ty_index }, - .tag = .type_index_leb, - .addend = 0, - }); - code.appendNTimesAssumeCapacity(0, 5); - } else { - const index: Wasm.Flush.FuncTypeIndex = .fromTypeIndex(func_ty_index, &wasm.flush_buffer); - writeUleb128(code, @intFromEnum(index)); - } - writeUleb128(code, @as(u32, 0)); // table index - - inst += 1; - continue :loop tags[inst]; - }, - - .call_tag_name => { - try code.ensureUnusedCapacity(gpa, 6); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.tagNameSymbolIndex(datas[inst].ip_index) }, - .tag = .function_index_leb, - .addend = 0, - }); - code.appendNTimesAssumeCapacity(0, 5); - } else { - appendOutputFunctionIndex(code, .fromTagNameType(wasm, datas[inst].ip_index)); - } - - inst += 1; - continue :loop tags[inst]; - }, - - .call_intrinsic => { - // Although this currently uses `wasm.internString`, note that it - // *could* be changed to directly index into a preloaded strings - // table initialized based on the `Mir.Intrinsic` enum. - const symbol_name = try wasm.internString(@tagName(datas[inst].intrinsic)); - - try code.ensureUnusedCapacity(gpa, 6); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.symbolNameIndex(symbol_name) }, - .tag = .function_index_leb, - .addend = 0, - }); - code.appendNTimesAssumeCapacity(0, 5); - } else { - appendOutputFunctionIndex(code, .fromSymbolName(wasm, symbol_name)); - } - - inst += 1; - continue :loop tags[inst]; - }, - - .global_set_sp => { - try code.ensureUnusedCapacity(gpa, 6); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.global_set)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.stackPointerSymbolIndex() }, - .tag = .global_index_leb, - .addend = 0, - }); - code.appendNTimesAssumeCapacity(0, 5); - } else { - const sp_global: Wasm.GlobalIndex = .stack_pointer; - writeUleb128(code, @intFromEnum(sp_global)); - } - - inst += 1; - continue :loop tags[inst]; - }, - - .f32_const => { - try code.ensureUnusedCapacity(gpa, 5); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.f32_const)); - std.mem.writeInt(u32, code.addManyAsArrayAssumeCapacity(4), @bitCast(datas[inst].float32), .little); - - inst += 1; - continue :loop tags[inst]; - }, - - .f64_const => { - try code.ensureUnusedCapacity(gpa, 9); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.f64_const)); - const float64 = mir.extraData(Mir.Float64, datas[inst].payload).data; - std.mem.writeInt(u64, code.addManyAsArrayAssumeCapacity(8), float64.toInt(), .little); - - inst += 1; - continue :loop tags[inst]; - }, - .i32_const => { - try code.ensureUnusedCapacity(gpa, 6); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i32_const)); - writeSleb128(code, datas[inst].imm32); - - inst += 1; - continue :loop tags[inst]; - }, - .i64_const => { - try code.ensureUnusedCapacity(gpa, 11); - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i64_const)); - const int64: i64 = @bitCast(mir.extraData(Mir.Imm64, datas[inst].payload).data.toInt()); - writeSleb128(code, int64); - - inst += 1; - continue :loop tags[inst]; - }, - - .i32_load, - .i64_load, - .f32_load, - .f64_load, - .i32_load8_s, - .i32_load8_u, - .i32_load16_s, - .i32_load16_u, - .i64_load8_s, - .i64_load8_u, - .i64_load16_s, - .i64_load16_u, - .i64_load32_s, - .i64_load32_u, - .i32_store, - .i64_store, - .f32_store, - .f64_store, - .i32_store8, - .i32_store16, - .i64_store8, - .i64_store16, - .i64_store32, - => { - try code.ensureUnusedCapacity(gpa, 1 + 20); - code.appendAssumeCapacity(@intFromEnum(tags[inst])); - encodeMemArg(code, mir.extraData(Mir.MemArg, datas[inst].payload).data); - inst += 1; - continue :loop tags[inst]; - }, - - .end, - .@"return", - .@"unreachable", - .select, - .i32_eqz, - .i32_eq, - .i32_ne, - .i32_lt_s, - .i32_lt_u, - .i32_gt_s, - .i32_gt_u, - .i32_le_s, - .i32_le_u, - .i32_ge_s, - .i32_ge_u, - .i64_eqz, - .i64_eq, - .i64_ne, - .i64_lt_s, - .i64_lt_u, - .i64_gt_s, - .i64_gt_u, - .i64_le_s, - .i64_le_u, - .i64_ge_s, - .i64_ge_u, - .f32_eq, - .f32_ne, - .f32_lt, - .f32_gt, - .f32_le, - .f32_ge, - .f64_eq, - .f64_ne, - .f64_lt, - .f64_gt, - .f64_le, - .f64_ge, - .i32_add, - .i32_sub, - .i32_mul, - .i32_div_s, - .i32_div_u, - .i32_and, - .i32_or, - .i32_xor, - .i32_shl, - .i32_shr_s, - .i32_shr_u, - .i64_add, - .i64_sub, - .i64_mul, - .i64_div_s, - .i64_div_u, - .i64_and, - .i64_or, - .i64_xor, - .i64_shl, - .i64_shr_s, - .i64_shr_u, - .f32_abs, - .f32_neg, - .f32_ceil, - .f32_floor, - .f32_trunc, - .f32_nearest, - .f32_sqrt, - .f32_add, - .f32_sub, - .f32_mul, - .f32_div, - .f32_min, - .f32_max, - .f32_copysign, - .f64_abs, - .f64_neg, - .f64_ceil, - .f64_floor, - .f64_trunc, - .f64_nearest, - .f64_sqrt, - .f64_add, - .f64_sub, - .f64_mul, - .f64_div, - .f64_min, - .f64_max, - .f64_copysign, - .i32_wrap_i64, - .i64_extend_i32_s, - .i64_extend_i32_u, - .i32_extend8_s, - .i32_extend16_s, - .i64_extend8_s, - .i64_extend16_s, - .i64_extend32_s, - .f32_demote_f64, - .f64_promote_f32, - .i32_reinterpret_f32, - .i64_reinterpret_f64, - .f32_reinterpret_i32, - .f64_reinterpret_i64, - .i32_trunc_f32_s, - .i32_trunc_f32_u, - .i32_trunc_f64_s, - .i32_trunc_f64_u, - .i64_trunc_f32_s, - .i64_trunc_f32_u, - .i64_trunc_f64_s, - .i64_trunc_f64_u, - .f32_convert_i32_s, - .f32_convert_i32_u, - .f32_convert_i64_s, - .f32_convert_i64_u, - .f64_convert_i32_s, - .f64_convert_i32_u, - .f64_convert_i64_s, - .f64_convert_i64_u, - .i32_rem_s, - .i32_rem_u, - .i64_rem_s, - .i64_rem_u, - .i32_popcnt, - .i64_popcnt, - .i32_clz, - .i32_ctz, - .i64_clz, - .i64_ctz, - => { - try code.append(gpa, @intFromEnum(tags[inst])); - inst += 1; - continue :loop tags[inst]; - }, - - .misc_prefix => { - try code.ensureUnusedCapacity(gpa, 6 + 6); - const extra_index = datas[inst].payload; - const opcode = mir.extra[extra_index]; - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.misc_prefix)); - writeUleb128(code, opcode); - switch (@as(std.wasm.MiscOpcode, @enumFromInt(opcode))) { - // bulk-memory opcodes - .data_drop => { - const segment = mir.extra[extra_index + 1]; - writeUleb128(code, segment); - - inst += 1; - continue :loop tags[inst]; - }, - .memory_init => { - const segment = mir.extra[extra_index + 1]; - writeUleb128(code, segment); - writeUleb128(code, @as(u32, 0)); // memory index - - inst += 1; - continue :loop tags[inst]; - }, - .memory_fill => { - writeUleb128(code, @as(u32, 0)); // memory index - - inst += 1; - continue :loop tags[inst]; - }, - .memory_copy => { - writeUleb128(code, @as(u32, 0)); // dst memory index - writeUleb128(code, @as(u32, 0)); // src memory index - - inst += 1; - continue :loop tags[inst]; - }, - - // nontrapping-float-to-int-conversion opcodes - .i32_trunc_sat_f32_s, - .i32_trunc_sat_f32_u, - .i32_trunc_sat_f64_s, - .i32_trunc_sat_f64_u, - .i64_trunc_sat_f32_s, - .i64_trunc_sat_f32_u, - .i64_trunc_sat_f64_s, - .i64_trunc_sat_f64_u, - => { - inst += 1; - continue :loop tags[inst]; - }, - - .table_init => @panic("TODO"), - .elem_drop => @panic("TODO"), - .table_copy => @panic("TODO"), - .table_grow => @panic("TODO"), - .table_size => @panic("TODO"), - .table_fill => @panic("TODO"), - - _ => unreachable, - } - comptime unreachable; - }, - .simd_prefix => { - try code.ensureUnusedCapacity(gpa, 6 + 20); - const extra_index = datas[inst].payload; - const opcode = mir.extra[extra_index]; - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.simd_prefix)); - writeUleb128(code, opcode); - switch (@as(std.wasm.SimdOpcode, @enumFromInt(opcode))) { - .v128_store, - .v128_load, - .v128_load8_splat, - .v128_load16_splat, - .v128_load32_splat, - .v128_load64_splat, - => { - encodeMemArg(code, mir.extraData(Mir.MemArg, extra_index + 1).data); - inst += 1; - continue :loop tags[inst]; - }, - .v128_const, .i8x16_shuffle => { - code.appendSliceAssumeCapacity(std.mem.asBytes(mir.extra[extra_index + 1 ..][0..4])); - inst += 1; - continue :loop tags[inst]; - }, - .i8x16_extract_lane_s, - .i8x16_extract_lane_u, - .i8x16_replace_lane, - .i16x8_extract_lane_s, - .i16x8_extract_lane_u, - .i16x8_replace_lane, - .i32x4_extract_lane, - .i32x4_replace_lane, - .i64x2_extract_lane, - .i64x2_replace_lane, - .f32x4_extract_lane, - .f32x4_replace_lane, - .f64x2_extract_lane, - .f64x2_replace_lane, - => { - code.appendAssumeCapacity(@intCast(mir.extra[extra_index + 1])); - inst += 1; - continue :loop tags[inst]; - }, - .i8x16_splat, - .i16x8_splat, - .i32x4_splat, - .i64x2_splat, - .f32x4_splat, - .f64x2_splat, - => { - inst += 1; - continue :loop tags[inst]; - }, - - .v128_load8x8_s => @panic("TODO"), - .v128_load8x8_u => @panic("TODO"), - .v128_load16x4_s => @panic("TODO"), - .v128_load16x4_u => @panic("TODO"), - .v128_load32x2_s => @panic("TODO"), - .v128_load32x2_u => @panic("TODO"), - .i8x16_swizzle => @panic("TODO"), - .i8x16_eq => @panic("TODO"), - .i16x8_eq => @panic("TODO"), - .i32x4_eq => @panic("TODO"), - .i8x16_ne => @panic("TODO"), - .i16x8_ne => @panic("TODO"), - .i32x4_ne => @panic("TODO"), - .i8x16_lt_s => @panic("TODO"), - .i16x8_lt_s => @panic("TODO"), - .i32x4_lt_s => @panic("TODO"), - .i8x16_lt_u => @panic("TODO"), - .i16x8_lt_u => @panic("TODO"), - .i32x4_lt_u => @panic("TODO"), - .i8x16_gt_s => @panic("TODO"), - .i16x8_gt_s => @panic("TODO"), - .i32x4_gt_s => @panic("TODO"), - .i8x16_gt_u => @panic("TODO"), - .i16x8_gt_u => @panic("TODO"), - .i32x4_gt_u => @panic("TODO"), - .i8x16_le_s => @panic("TODO"), - .i16x8_le_s => @panic("TODO"), - .i32x4_le_s => @panic("TODO"), - .i8x16_le_u => @panic("TODO"), - .i16x8_le_u => @panic("TODO"), - .i32x4_le_u => @panic("TODO"), - .i8x16_ge_s => @panic("TODO"), - .i16x8_ge_s => @panic("TODO"), - .i32x4_ge_s => @panic("TODO"), - .i8x16_ge_u => @panic("TODO"), - .i16x8_ge_u => @panic("TODO"), - .i32x4_ge_u => @panic("TODO"), - .f32x4_eq => @panic("TODO"), - .f64x2_eq => @panic("TODO"), - .f32x4_ne => @panic("TODO"), - .f64x2_ne => @panic("TODO"), - .f32x4_lt => @panic("TODO"), - .f64x2_lt => @panic("TODO"), - .f32x4_gt => @panic("TODO"), - .f64x2_gt => @panic("TODO"), - .f32x4_le => @panic("TODO"), - .f64x2_le => @panic("TODO"), - .f32x4_ge => @panic("TODO"), - .f64x2_ge => @panic("TODO"), - .v128_not => @panic("TODO"), - .v128_and => @panic("TODO"), - .v128_andnot => @panic("TODO"), - .v128_or => @panic("TODO"), - .v128_xor => @panic("TODO"), - .v128_bitselect => @panic("TODO"), - .v128_any_true => @panic("TODO"), - .v128_load8_lane => @panic("TODO"), - .v128_load16_lane => @panic("TODO"), - .v128_load32_lane => @panic("TODO"), - .v128_load64_lane => @panic("TODO"), - .v128_store8_lane => @panic("TODO"), - .v128_store16_lane => @panic("TODO"), - .v128_store32_lane => @panic("TODO"), - .v128_store64_lane => @panic("TODO"), - .v128_load32_zero => @panic("TODO"), - .v128_load64_zero => @panic("TODO"), - .f32x4_demote_f64x2_zero => @panic("TODO"), - .f64x2_promote_low_f32x4 => @panic("TODO"), - .i8x16_abs => @panic("TODO"), - .i16x8_abs => @panic("TODO"), - .i32x4_abs => @panic("TODO"), - .i64x2_abs => @panic("TODO"), - .i8x16_neg => @panic("TODO"), - .i16x8_neg => @panic("TODO"), - .i32x4_neg => @panic("TODO"), - .i64x2_neg => @panic("TODO"), - .i8x16_popcnt => @panic("TODO"), - .i16x8_q15mulr_sat_s => @panic("TODO"), - .i8x16_all_true => @panic("TODO"), - .i16x8_all_true => @panic("TODO"), - .i32x4_all_true => @panic("TODO"), - .i64x2_all_true => @panic("TODO"), - .i8x16_bitmask => @panic("TODO"), - .i16x8_bitmask => @panic("TODO"), - .i32x4_bitmask => @panic("TODO"), - .i64x2_bitmask => @panic("TODO"), - .i8x16_narrow_i16x8_s => @panic("TODO"), - .i16x8_narrow_i32x4_s => @panic("TODO"), - .i8x16_narrow_i16x8_u => @panic("TODO"), - .i16x8_narrow_i32x4_u => @panic("TODO"), - .f32x4_ceil => @panic("TODO"), - .i16x8_extend_low_i8x16_s => @panic("TODO"), - .i32x4_extend_low_i16x8_s => @panic("TODO"), - .i64x2_extend_low_i32x4_s => @panic("TODO"), - .f32x4_floor => @panic("TODO"), - .i16x8_extend_high_i8x16_s => @panic("TODO"), - .i32x4_extend_high_i16x8_s => @panic("TODO"), - .i64x2_extend_high_i32x4_s => @panic("TODO"), - .f32x4_trunc => @panic("TODO"), - .i16x8_extend_low_i8x16_u => @panic("TODO"), - .i32x4_extend_low_i16x8_u => @panic("TODO"), - .i64x2_extend_low_i32x4_u => @panic("TODO"), - .f32x4_nearest => @panic("TODO"), - .i16x8_extend_high_i8x16_u => @panic("TODO"), - .i32x4_extend_high_i16x8_u => @panic("TODO"), - .i64x2_extend_high_i32x4_u => @panic("TODO"), - .i8x16_shl => @panic("TODO"), - .i16x8_shl => @panic("TODO"), - .i32x4_shl => @panic("TODO"), - .i64x2_shl => @panic("TODO"), - .i8x16_shr_s => @panic("TODO"), - .i16x8_shr_s => @panic("TODO"), - .i32x4_shr_s => @panic("TODO"), - .i64x2_shr_s => @panic("TODO"), - .i8x16_shr_u => @panic("TODO"), - .i16x8_shr_u => @panic("TODO"), - .i32x4_shr_u => @panic("TODO"), - .i64x2_shr_u => @panic("TODO"), - .i8x16_add => @panic("TODO"), - .i16x8_add => @panic("TODO"), - .i32x4_add => @panic("TODO"), - .i64x2_add => @panic("TODO"), - .i8x16_add_sat_s => @panic("TODO"), - .i16x8_add_sat_s => @panic("TODO"), - .i8x16_add_sat_u => @panic("TODO"), - .i16x8_add_sat_u => @panic("TODO"), - .i8x16_sub => @panic("TODO"), - .i16x8_sub => @panic("TODO"), - .i32x4_sub => @panic("TODO"), - .i64x2_sub => @panic("TODO"), - .i8x16_sub_sat_s => @panic("TODO"), - .i16x8_sub_sat_s => @panic("TODO"), - .i8x16_sub_sat_u => @panic("TODO"), - .i16x8_sub_sat_u => @panic("TODO"), - .f64x2_ceil => @panic("TODO"), - .f64x2_nearest => @panic("TODO"), - .f64x2_floor => @panic("TODO"), - .i16x8_mul => @panic("TODO"), - .i32x4_mul => @panic("TODO"), - .i64x2_mul => @panic("TODO"), - .i8x16_min_s => @panic("TODO"), - .i16x8_min_s => @panic("TODO"), - .i32x4_min_s => @panic("TODO"), - .i64x2_eq => @panic("TODO"), - .i8x16_min_u => @panic("TODO"), - .i16x8_min_u => @panic("TODO"), - .i32x4_min_u => @panic("TODO"), - .i64x2_ne => @panic("TODO"), - .i8x16_max_s => @panic("TODO"), - .i16x8_max_s => @panic("TODO"), - .i32x4_max_s => @panic("TODO"), - .i64x2_lt_s => @panic("TODO"), - .i8x16_max_u => @panic("TODO"), - .i16x8_max_u => @panic("TODO"), - .i32x4_max_u => @panic("TODO"), - .i64x2_gt_s => @panic("TODO"), - .f64x2_trunc => @panic("TODO"), - .i32x4_dot_i16x8_s => @panic("TODO"), - .i64x2_le_s => @panic("TODO"), - .i8x16_avgr_u => @panic("TODO"), - .i16x8_avgr_u => @panic("TODO"), - .i64x2_ge_s => @panic("TODO"), - .i16x8_extadd_pairwise_i8x16_s => @panic("TODO"), - .i16x8_extmul_low_i8x16_s => @panic("TODO"), - .i32x4_extmul_low_i16x8_s => @panic("TODO"), - .i64x2_extmul_low_i32x4_s => @panic("TODO"), - .i16x8_extadd_pairwise_i8x16_u => @panic("TODO"), - .i16x8_extmul_high_i8x16_s => @panic("TODO"), - .i32x4_extmul_high_i16x8_s => @panic("TODO"), - .i64x2_extmul_high_i32x4_s => @panic("TODO"), - .i32x4_extadd_pairwise_i16x8_s => @panic("TODO"), - .i16x8_extmul_low_i8x16_u => @panic("TODO"), - .i32x4_extmul_low_i16x8_u => @panic("TODO"), - .i64x2_extmul_low_i32x4_u => @panic("TODO"), - .i32x4_extadd_pairwise_i16x8_u => @panic("TODO"), - .i16x8_extmul_high_i8x16_u => @panic("TODO"), - .i32x4_extmul_high_i16x8_u => @panic("TODO"), - .i64x2_extmul_high_i32x4_u => @panic("TODO"), - .f32x4_abs => @panic("TODO"), - .f64x2_abs => @panic("TODO"), - .f32x4_neg => @panic("TODO"), - .f64x2_neg => @panic("TODO"), - .f32x4_sqrt => @panic("TODO"), - .f64x2_sqrt => @panic("TODO"), - .f32x4_add => @panic("TODO"), - .f64x2_add => @panic("TODO"), - .f32x4_sub => @panic("TODO"), - .f64x2_sub => @panic("TODO"), - .f32x4_mul => @panic("TODO"), - .f64x2_mul => @panic("TODO"), - .f32x4_div => @panic("TODO"), - .f64x2_div => @panic("TODO"), - .f32x4_min => @panic("TODO"), - .f64x2_min => @panic("TODO"), - .f32x4_max => @panic("TODO"), - .f64x2_max => @panic("TODO"), - .f32x4_pmin => @panic("TODO"), - .f64x2_pmin => @panic("TODO"), - .f32x4_pmax => @panic("TODO"), - .f64x2_pmax => @panic("TODO"), - .i32x4_trunc_sat_f32x4_s => @panic("TODO"), - .i32x4_trunc_sat_f32x4_u => @panic("TODO"), - .f32x4_convert_i32x4_s => @panic("TODO"), - .f32x4_convert_i32x4_u => @panic("TODO"), - .i32x4_trunc_sat_f64x2_s_zero => @panic("TODO"), - .i32x4_trunc_sat_f64x2_u_zero => @panic("TODO"), - .f64x2_convert_low_i32x4_s => @panic("TODO"), - .f64x2_convert_low_i32x4_u => @panic("TODO"), - .i8x16_relaxed_swizzle => @panic("TODO"), - .i32x4_relaxed_trunc_f32x4_s => @panic("TODO"), - .i32x4_relaxed_trunc_f32x4_u => @panic("TODO"), - .i32x4_relaxed_trunc_f64x2_s_zero => @panic("TODO"), - .i32x4_relaxed_trunc_f64x2_u_zero => @panic("TODO"), - .f32x4_relaxed_madd => @panic("TODO"), - .f32x4_relaxed_nmadd => @panic("TODO"), - .f64x2_relaxed_madd => @panic("TODO"), - .f64x2_relaxed_nmadd => @panic("TODO"), - .i8x16_relaxed_laneselect => @panic("TODO"), - .i16x8_relaxed_laneselect => @panic("TODO"), - .i32x4_relaxed_laneselect => @panic("TODO"), - .i64x2_relaxed_laneselect => @panic("TODO"), - .f32x4_relaxed_min => @panic("TODO"), - .f32x4_relaxed_max => @panic("TODO"), - .f64x2_relaxed_min => @panic("TODO"), - .f64x2_relaxed_max => @panic("TODO"), - .i16x8_relaxed_q15mulr_s => @panic("TODO"), - .i16x8_relaxed_dot_i8x16_i7x16_s => @panic("TODO"), - .i32x4_relaxed_dot_i8x16_i7x16_add_s => @panic("TODO"), - .f32x4_relaxed_dot_bf16x8_add_f32x4 => @panic("TODO"), - } - comptime unreachable; - }, - .atomics_prefix => { - try code.ensureUnusedCapacity(gpa, 6 + 20); - - const extra_index = datas[inst].payload; - const opcode = mir.extra[extra_index]; - code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.atomics_prefix)); - writeUleb128(code, opcode); - switch (@as(std.wasm.AtomicsOpcode, @enumFromInt(opcode))) { - .i32_atomic_load, - .i64_atomic_load, - .i32_atomic_load8_u, - .i32_atomic_load16_u, - .i64_atomic_load8_u, - .i64_atomic_load16_u, - .i64_atomic_load32_u, - .i32_atomic_store, - .i64_atomic_store, - .i32_atomic_store8, - .i32_atomic_store16, - .i64_atomic_store8, - .i64_atomic_store16, - .i64_atomic_store32, - .i32_atomic_rmw_add, - .i64_atomic_rmw_add, - .i32_atomic_rmw8_add_u, - .i32_atomic_rmw16_add_u, - .i64_atomic_rmw8_add_u, - .i64_atomic_rmw16_add_u, - .i64_atomic_rmw32_add_u, - .i32_atomic_rmw_sub, - .i64_atomic_rmw_sub, - .i32_atomic_rmw8_sub_u, - .i32_atomic_rmw16_sub_u, - .i64_atomic_rmw8_sub_u, - .i64_atomic_rmw16_sub_u, - .i64_atomic_rmw32_sub_u, - .i32_atomic_rmw_and, - .i64_atomic_rmw_and, - .i32_atomic_rmw8_and_u, - .i32_atomic_rmw16_and_u, - .i64_atomic_rmw8_and_u, - .i64_atomic_rmw16_and_u, - .i64_atomic_rmw32_and_u, - .i32_atomic_rmw_or, - .i64_atomic_rmw_or, - .i32_atomic_rmw8_or_u, - .i32_atomic_rmw16_or_u, - .i64_atomic_rmw8_or_u, - .i64_atomic_rmw16_or_u, - .i64_atomic_rmw32_or_u, - .i32_atomic_rmw_xor, - .i64_atomic_rmw_xor, - .i32_atomic_rmw8_xor_u, - .i32_atomic_rmw16_xor_u, - .i64_atomic_rmw8_xor_u, - .i64_atomic_rmw16_xor_u, - .i64_atomic_rmw32_xor_u, - .i32_atomic_rmw_xchg, - .i64_atomic_rmw_xchg, - .i32_atomic_rmw8_xchg_u, - .i32_atomic_rmw16_xchg_u, - .i64_atomic_rmw8_xchg_u, - .i64_atomic_rmw16_xchg_u, - .i64_atomic_rmw32_xchg_u, - - .i32_atomic_rmw_cmpxchg, - .i64_atomic_rmw_cmpxchg, - .i32_atomic_rmw8_cmpxchg_u, - .i32_atomic_rmw16_cmpxchg_u, - .i64_atomic_rmw8_cmpxchg_u, - .i64_atomic_rmw16_cmpxchg_u, - .i64_atomic_rmw32_cmpxchg_u, - => { - const mem_arg = mir.extraData(Mir.MemArg, extra_index + 1).data; - encodeMemArg(code, mem_arg); - inst += 1; - continue :loop tags[inst]; - }, - .atomic_fence => { - // Hard-codes memory index 0 since multi-memory proposal is - // not yet accepted nor implemented. - const memory_index: u32 = 0; - writeUleb128(code, memory_index); - inst += 1; - continue :loop tags[inst]; - }, - .memory_atomic_notify => @panic("TODO"), - .memory_atomic_wait32 => @panic("TODO"), - .memory_atomic_wait64 => @panic("TODO"), - } - comptime unreachable; - }, - } - comptime unreachable; -} - -/// Asserts 20 unused capacity. -fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void { - assert(code.unusedCapacitySlice().len >= 20); - // Wasm encodes alignment as power of 2, rather than natural alignment. - const encoded_alignment = @ctz(mem_arg.alignment); - writeUleb128(code, encoded_alignment); - writeUleb128(code, mem_arg.offset); -} - -fn uavRefObj(wasm: *Wasm, code: *ArrayList(u8), value: InternPool.Index, offset: i32, is_wasm32: bool) !void { - const comp = wasm.base.comp; - const gpa = comp.gpa; - const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; - - try code.ensureUnusedCapacity(gpa, 11); - code.appendAssumeCapacity(@intFromEnum(opcode)); - - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.uavSymbolIndex(value) }, - .tag = if (is_wasm32) .memory_addr_leb else .memory_addr_leb64, - .addend = offset, - }); - code.appendNTimesAssumeCapacity(0, if (is_wasm32) 5 else 10); -} - -fn uavRefExe(wasm: *Wasm, code: *ArrayList(u8), value: InternPool.Index, offset: i32, is_wasm32: bool) !void { - const comp = wasm.base.comp; - const gpa = comp.gpa; - const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; - - try code.ensureUnusedCapacity(gpa, 11); - code.appendAssumeCapacity(@intFromEnum(opcode)); - - const addr = wasm.uavAddr(value); - writeUleb128(code, @as(u32, @intCast(@as(i64, addr) + offset))); -} - -fn navRefOff(wasm: *Wasm, code: *ArrayList(u8), data: Mir.NavRefOff, is_wasm32: bool) !void { - const comp = wasm.base.comp; - const zcu = comp.zcu.?; - const ip = &zcu.intern_pool; - const gpa = comp.gpa; - const is_obj = comp.config.output_mode == .Obj; - const nav_ty = ip.getNav(data.nav_index).typeOf(ip); - assert(!ip.isFunctionType(nav_ty)); - - try code.ensureUnusedCapacity(gpa, 11); - - const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; - code.appendAssumeCapacity(@intFromEnum(opcode)); - if (is_obj) { - try wasm.out_relocs.append(gpa, .{ - .offset = @intCast(code.items.len), - .pointee = .{ .symbol_index = try wasm.navSymbolIndex(data.nav_index) }, - .tag = if (is_wasm32) .memory_addr_leb else .memory_addr_leb64, - .addend = data.offset, - }); - code.appendNTimesAssumeCapacity(0, if (is_wasm32) 5 else 10); - } else { - const addr = wasm.navAddr(data.nav_index); - writeUleb128(code, @as(u32, @intCast(@as(i64, addr) + data.offset))); - } -} - -fn appendOutputFunctionIndex(code: *ArrayList(u8), i: Wasm.OutputFunctionIndex) void { - writeUleb128(code, @intFromEnum(i)); -} - -fn writeUleb128(code: *ArrayList(u8), arg: anytype) void { - var w: std.Io.Writer = .fixed(code.unusedCapacitySlice()); - w.writeUleb128(arg) catch unreachable; - code.items.len += w.end; -} - -fn writeSleb128(code: *ArrayList(u8), arg: anytype) void { - var w: std.Io.Writer = .fixed(code.unusedCapacitySlice()); - w.writeSleb128(arg) catch unreachable; - code.items.len += w.end; -} diff --git a/src/arch/wasm/Mir.zig b/src/arch/wasm/Mir.zig deleted file mode 100644 index af6e0eb40b590cfe486c246410b455af582f73d3..0000000000000000000000000000000000000000 --- a/src/arch/wasm/Mir.zig +++ /dev/null @@ -1,1003 +0,0 @@ -//! Machine Intermediate Representation. -//! This representation is produced by wasm Codegen. -//! Each of these instructions have a 1:1 mapping to a wasm opcode, -//! but may contain metadata for a specific opcode such as an immediate. -//! MIR can be lowered to both textual code (wat) and binary format (wasm). -//! The main benefits of MIR is optimization passes, pre-allocated locals, -//! and known jump labels for blocks. - -const Mir = @This(); -const InternPool = @import("../../InternPool.zig"); -const Wasm = @import("../../link/Wasm.zig"); -const Emit = @import("Emit.zig"); -const Alignment = InternPool.Alignment; - -const builtin = @import("builtin"); -const std = @import("std"); -const assert = std.debug.assert; -const leb = std.leb; - -instructions: std.MultiArrayList(Inst).Slice, -/// A slice of indexes where the meaning of the data is determined by the -/// `Inst.Tag` value. -extra: []const u32, -locals: []const std.wasm.Valtype, -prologue: Prologue, - -/// Not directly used by `Emit`, but the linker needs this to merge it with a global set. -/// Value is the explicit alignment if greater than natural alignment, `.none` otherwise. -uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment), -/// Not directly used by `Emit`, but the linker needs this to merge it with a global set. -indirect_function_set: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, void), -/// Not directly used by `Emit`, but the linker needs this to ensure these types are interned. -func_tys: std.AutoArrayHashMapUnmanaged(InternPool.Index, void), -/// Not directly used by `Emit`, but the linker needs this to add it to its own refcount. -error_name_table_ref_count: u32, - -pub const Prologue = extern struct { - flags: Flags, - sp_local: u32, - stack_size: u32, - bottom_stack_local: u32, - - pub const Flags = packed struct(u32) { - stack_alignment: Alignment, - padding: u26 = 0, - }; - - pub const none: Prologue = .{ - .sp_local = 0, - .flags = .{ .stack_alignment = .none }, - .stack_size = 0, - .bottom_stack_local = 0, - }; - - pub fn isNone(p: *const Prologue) bool { - return p.flags.stack_alignment != .none; - } -}; - -pub const Inst = struct { - /// The opcode that represents this instruction - tag: Tag, - /// Data is determined by the set `tag`. - /// For example, `data` will be an i32 for when `tag` is 'i32_const'. - data: Data, - - /// The position of a given MIR isntruction with the instruction list. - pub const Index = u32; - - /// Some tags match wasm opcode values to facilitate trivial lowering. - pub const Tag = enum(u8) { - /// Uses `tag`. - @"unreachable" = 0x00, - /// Emits epilogue begin debug information. Marks the end of the function. - /// - /// Uses `tag` (no additional data). - dbg_epilogue_begin, - /// Creates a new block that can be jump from. - /// - /// Type of the block is given in data `block_type` - block = 0x02, - /// Creates a new loop. - /// - /// Type of the loop is given in data `block_type` - loop = 0x03, - /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the - /// memory address of an unnamed constant. When emitting an object - /// file, this adds a relocation. - /// - /// This may not refer to a function. - /// - /// Uses `ip_index`. - uav_ref, - /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the - /// memory address of an unnamed constant, offset by an integer value. - /// When emitting an object file, this adds a relocation. - /// - /// This may not refer to a function. - /// - /// Uses `payload` pointing to a `UavRefOff`. - uav_ref_off, - /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the - /// memory address of a named constant. - /// - /// May not refer to a function. - /// - /// Uses `nav_index`. - nav_ref, - /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the - /// memory address of named constant, offset by an integer value. - /// When emitting an object file, this adds a relocation. - /// - /// May not refer to a function. - /// - /// Uses `payload` pointing to a `NavRefOff`. - nav_ref_off, - /// Lowers to an i32_const which is the index of the function in the - /// table section. - /// - /// Uses `nav_index`. - func_ref, - /// Inserts debug information about the current line and column - /// of the source code - /// - /// Uses `payload` of which the payload type is `DbgLineColumn` - dbg_line, - /// Lowers to an i32_const containing the number of unique Zig error - /// names. - /// Uses `tag`. - errors_len, - /// Represents the end of a function body or an initialization expression - /// - /// Uses `tag` (no additional data). - end = 0x0B, - /// Breaks from the current block to a label - /// - /// Uses `label` where index represents the label to jump to - br = 0x0C, - /// Breaks from the current block if the stack value is non-zero - /// - /// Uses `label` where index represents the label to jump to - br_if = 0x0D, - /// Jump table that takes the stack value as an index where each value - /// represents the label to jump to. - /// - /// Data is extra of which the Payload's type is `JumpTable` - br_table, - /// Returns from the function - /// - /// Uses `tag`. - @"return" = 0x0F, - /// Lowers to an i32_const (wasm32) or i64_const (wasm64) containing - /// the base address of the table of error code names, with each - /// element being a null-terminated slice. - /// - /// Uses `tag`. - error_name_table_ref, - /// Calls a function using `nav_index`. - call_nav, - /// Calls a function pointer by its function signature - /// and index into the function table. - /// - /// Uses `ip_index`; the `InternPool.Index` is the function type. - call_indirect, - /// Calls a function by its index. - /// - /// The function is the auto-generated tag name function for the type - /// provided in `ip_index`. - call_tag_name, - /// Lowers to a `call` instruction, using `intrinsic`. - call_intrinsic, - /// Pops three values from the stack and pushes - /// the first or second value dependent on the third value. - /// Uses `tag` - select = 0x1B, - /// Loads a local at given index onto the stack. - /// - /// Uses `label` - local_get = 0x20, - /// Pops a value from the stack into the local at given index. - /// Stack value must be of the same type as the local. - /// - /// Uses `label` - local_set = 0x21, - /// Sets a local at given index using the value at the top of the stack without popping the value. - /// Stack value must have the same type as the local. - /// - /// Uses `label` - local_tee = 0x22, - /// Pops a value from the stack and sets the stack pointer global. - /// The value must be the same type as the stack pointer global. - /// - /// Uses `tag` (no additional data). - global_set_sp, - /// Loads a 32-bit integer from memory (data section) onto the stack - /// Pops the value from the stack which represents the offset into memory. - /// - /// Uses `payload` of type `MemArg`. - i32_load = 0x28, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load = 0x29, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - f32_load = 0x2A, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - f64_load = 0x2B, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i32_load8_s = 0x2C, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i32_load8_u = 0x2D, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i32_load16_s = 0x2E, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i32_load16_u = 0x2F, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load8_s = 0x30, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load8_u = 0x31, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load16_s = 0x32, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load16_u = 0x33, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load32_s = 0x34, - /// Loads a value from memory onto the stack, based on the signedness - /// and bitsize of the type. - /// - /// Uses `payload` with type `MemArg` - i64_load32_u = 0x35, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `payload` of type `MemArg`. - i32_store = 0x36, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - i64_store = 0x37, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - f32_store = 0x38, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - f64_store = 0x39, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - i32_store8 = 0x3A, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - i32_store16 = 0x3B, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - i64_store8 = 0x3C, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - i64_store16 = 0x3D, - /// Pops 2 values from the stack, where the first value represents the value to write into memory - /// and the second value represents the offset into memory where the value must be written to. - /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. - /// - /// Uses `Payload` with type `MemArg` - i64_store32 = 0x3E, - /// Returns the memory size in amount of pages. - /// - /// Uses `label` - memory_size = 0x3F, - /// Increases the memory by given number of pages. - /// - /// Uses `label` - memory_grow = 0x40, - /// Loads a 32-bit signed immediate value onto the stack - /// - /// Uses `imm32` - i32_const, - /// Loads a i64-bit signed immediate value onto the stack - /// - /// uses `payload` of type `Imm64` - i64_const, - /// Loads a 32-bit float value onto the stack. - /// - /// Uses `float32` - f32_const, - /// Loads a 64-bit float value onto the stack. - /// - /// Uses `payload` of type `Float64` - f64_const, - /// Uses `tag` - i32_eqz = 0x45, - /// Uses `tag` - i32_eq = 0x46, - /// Uses `tag` - i32_ne = 0x47, - /// Uses `tag` - i32_lt_s = 0x48, - /// Uses `tag` - i32_lt_u = 0x49, - /// Uses `tag` - i32_gt_s = 0x4A, - /// Uses `tag` - i32_gt_u = 0x4B, - /// Uses `tag` - i32_le_s = 0x4C, - /// Uses `tag` - i32_le_u = 0x4D, - /// Uses `tag` - i32_ge_s = 0x4E, - /// Uses `tag` - i32_ge_u = 0x4F, - /// Uses `tag` - i64_eqz = 0x50, - /// Uses `tag` - i64_eq = 0x51, - /// Uses `tag` - i64_ne = 0x52, - /// Uses `tag` - i64_lt_s = 0x53, - /// Uses `tag` - i64_lt_u = 0x54, - /// Uses `tag` - i64_gt_s = 0x55, - /// Uses `tag` - i64_gt_u = 0x56, - /// Uses `tag` - i64_le_s = 0x57, - /// Uses `tag` - i64_le_u = 0x58, - /// Uses `tag` - i64_ge_s = 0x59, - /// Uses `tag` - i64_ge_u = 0x5A, - /// Uses `tag` - f32_eq = 0x5B, - /// Uses `tag` - f32_ne = 0x5C, - /// Uses `tag` - f32_lt = 0x5D, - /// Uses `tag` - f32_gt = 0x5E, - /// Uses `tag` - f32_le = 0x5F, - /// Uses `tag` - f32_ge = 0x60, - /// Uses `tag` - f64_eq = 0x61, - /// Uses `tag` - f64_ne = 0x62, - /// Uses `tag` - f64_lt = 0x63, - /// Uses `tag` - f64_gt = 0x64, - /// Uses `tag` - f64_le = 0x65, - /// Uses `tag` - f64_ge = 0x66, - /// Uses `tag` - i32_clz = 0x67, - /// Uses `tag` - i32_ctz = 0x68, - /// Uses `tag` - i32_popcnt = 0x69, - /// Uses `tag` - i32_add = 0x6A, - /// Uses `tag` - i32_sub = 0x6B, - /// Uses `tag` - i32_mul = 0x6C, - /// Uses `tag` - i32_div_s = 0x6D, - /// Uses `tag` - i32_div_u = 0x6E, - /// Uses `tag` - i32_rem_s = 0x6F, - /// Uses `tag` - i32_rem_u = 0x70, - /// Uses `tag` - i32_and = 0x71, - /// Uses `tag` - i32_or = 0x72, - /// Uses `tag` - i32_xor = 0x73, - /// Uses `tag` - i32_shl = 0x74, - /// Uses `tag` - i32_shr_s = 0x75, - /// Uses `tag` - i32_shr_u = 0x76, - /// Uses `tag` - i64_clz = 0x79, - /// Uses `tag` - i64_ctz = 0x7A, - /// Uses `tag` - i64_popcnt = 0x7B, - /// Uses `tag` - i64_add = 0x7C, - /// Uses `tag` - i64_sub = 0x7D, - /// Uses `tag` - i64_mul = 0x7E, - /// Uses `tag` - i64_div_s = 0x7F, - /// Uses `tag` - i64_div_u = 0x80, - /// Uses `tag` - i64_rem_s = 0x81, - /// Uses `tag` - i64_rem_u = 0x82, - /// Uses `tag` - i64_and = 0x83, - /// Uses `tag` - i64_or = 0x84, - /// Uses `tag` - i64_xor = 0x85, - /// Uses `tag` - i64_shl = 0x86, - /// Uses `tag` - i64_shr_s = 0x87, - /// Uses `tag` - i64_shr_u = 0x88, - /// Uses `tag` - f32_abs = 0x8B, - /// Uses `tag` - f32_neg = 0x8C, - /// Uses `tag` - f32_ceil = 0x8D, - /// Uses `tag` - f32_floor = 0x8E, - /// Uses `tag` - f32_trunc = 0x8F, - /// Uses `tag` - f32_nearest = 0x90, - /// Uses `tag` - f32_sqrt = 0x91, - /// Uses `tag` - f32_add = 0x92, - /// Uses `tag` - f32_sub = 0x93, - /// Uses `tag` - f32_mul = 0x94, - /// Uses `tag` - f32_div = 0x95, - /// Uses `tag` - f32_min = 0x96, - /// Uses `tag` - f32_max = 0x97, - /// Uses `tag` - f32_copysign = 0x98, - /// Uses `tag` - f64_abs = 0x99, - /// Uses `tag` - f64_neg = 0x9A, - /// Uses `tag` - f64_ceil = 0x9B, - /// Uses `tag` - f64_floor = 0x9C, - /// Uses `tag` - f64_trunc = 0x9D, - /// Uses `tag` - f64_nearest = 0x9E, - /// Uses `tag` - f64_sqrt = 0x9F, - /// Uses `tag` - f64_add = 0xA0, - /// Uses `tag` - f64_sub = 0xA1, - /// Uses `tag` - f64_mul = 0xA2, - /// Uses `tag` - f64_div = 0xA3, - /// Uses `tag` - f64_min = 0xA4, - /// Uses `tag` - f64_max = 0xA5, - /// Uses `tag` - f64_copysign = 0xA6, - /// Uses `tag` - i32_wrap_i64 = 0xA7, - /// Uses `tag` - i32_trunc_f32_s = 0xA8, - /// Uses `tag` - i32_trunc_f32_u = 0xA9, - /// Uses `tag` - i32_trunc_f64_s = 0xAA, - /// Uses `tag` - i32_trunc_f64_u = 0xAB, - /// Uses `tag` - i64_extend_i32_s = 0xAC, - /// Uses `tag` - i64_extend_i32_u = 0xAD, - /// Uses `tag` - i64_trunc_f32_s = 0xAE, - /// Uses `tag` - i64_trunc_f32_u = 0xAF, - /// Uses `tag` - i64_trunc_f64_s = 0xB0, - /// Uses `tag` - i64_trunc_f64_u = 0xB1, - /// Uses `tag` - f32_convert_i32_s = 0xB2, - /// Uses `tag` - f32_convert_i32_u = 0xB3, - /// Uses `tag` - f32_convert_i64_s = 0xB4, - /// Uses `tag` - f32_convert_i64_u = 0xB5, - /// Uses `tag` - f32_demote_f64 = 0xB6, - /// Uses `tag` - f64_convert_i32_s = 0xB7, - /// Uses `tag` - f64_convert_i32_u = 0xB8, - /// Uses `tag` - f64_convert_i64_s = 0xB9, - /// Uses `tag` - f64_convert_i64_u = 0xBA, - /// Uses `tag` - f64_promote_f32 = 0xBB, - /// Uses `tag` - i32_reinterpret_f32 = 0xBC, - /// Uses `tag` - i64_reinterpret_f64 = 0xBD, - /// Uses `tag` - f32_reinterpret_i32 = 0xBE, - /// Uses `tag` - f64_reinterpret_i64 = 0xBF, - /// Uses `tag` - i32_extend8_s = 0xC0, - /// Uses `tag` - i32_extend16_s = 0xC1, - /// Uses `tag` - i64_extend8_s = 0xC2, - /// Uses `tag` - i64_extend16_s = 0xC3, - /// Uses `tag` - i64_extend32_s = 0xC4, - /// The instruction consists of a prefixed opcode. - /// The prefixed opcode can be found at payload's index. - /// - /// The `data` field depends on the extension instruction and - /// may contain additional data. - misc_prefix, - /// The instruction consists of a simd opcode. - /// The actual simd-opcode is found at payload's index. - /// - /// The `data` field depends on the simd instruction and - /// may contain additional data. - simd_prefix, - /// The instruction consists of an atomics opcode. - /// The actual atomics-opcode is found at payload's index. - /// - /// The `data` field depends on the atomics instruction and - /// may contain additional data. - atomics_prefix = 0xFE, - - /// From a given wasm opcode, returns a MIR tag. - pub fn fromOpcode(opcode: std.wasm.Opcode) Tag { - return @as(Tag, @enumFromInt(@intFromEnum(opcode))); // Given `Opcode` is not present as a tag for MIR yet - } - - /// Returns a wasm opcode from a given MIR tag. - pub fn toOpcode(self: Tag) std.wasm.Opcode { - return @as(std.wasm.Opcode, @enumFromInt(@intFromEnum(self))); - } - }; - - /// All instructions contain a 4-byte payload, which is contained within - /// this union. `Tag` determines which union tag is active, as well as - /// how to interpret the data within. - pub const Data = union { - /// Uses no additional data - tag: void, - /// Contains the result type of a block - block_type: std.wasm.BlockType, - /// Label: Each structured control instruction introduces an implicit label. - /// Labels are targets for branch instructions that reference them with - /// label indices. Unlike with other index spaces, indexing of labels - /// is relative by nesting depth, that is, label 0 refers to the - /// innermost structured control instruction enclosing the referring - /// branch instruction, while increasing indices refer to those farther - /// out. Consequently, labels can only be referenced from within the - /// associated structured control instruction. - label: u32, - /// Local: The index space for locals is only accessible inside a function and - /// includes the parameters of that function, which precede the local - /// variables. - local: u32, - /// A 32-bit immediate value. - imm32: i32, - /// A 32-bit float value - float32: f32, - /// Index into `extra`. Meaning of what can be found there is context-dependent. - payload: u32, - - ip_index: InternPool.Index, - nav_index: InternPool.Nav.Index, - intrinsic: Intrinsic, - - comptime { - switch (builtin.mode) { - .Debug, .ReleaseSafe => {}, - .ReleaseFast, .ReleaseSmall => assert(@sizeOf(Data) == 4), - } - } - }; -}; - -pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { - mir.instructions.deinit(gpa); - gpa.free(mir.extra); - gpa.free(mir.locals); - mir.uavs.deinit(gpa); - mir.indirect_function_set.deinit(gpa); - mir.func_tys.deinit(gpa); - mir.* = undefined; -} - -pub fn lower(mir: *const Mir, wasm: *Wasm, code: *std.ArrayListUnmanaged(u8)) std.mem.Allocator.Error!void { - const gpa = wasm.base.comp.gpa; - - // Write the locals in the prologue of the function body. - try code.ensureUnusedCapacity(gpa, 5 + mir.locals.len * 6 + 38); - - var w: std.Io.Writer = .fixed(code.unusedCapacitySlice()); - - w.writeLeb128(@as(u32, @intCast(mir.locals.len))) catch unreachable; - - for (mir.locals) |local| { - w.writeLeb128(@as(u32, 1)) catch unreachable; - w.writeByte(@intFromEnum(local)) catch unreachable; - } - - // Stack management section of function prologue. - const stack_alignment = mir.prologue.flags.stack_alignment; - if (stack_alignment.toByteUnits()) |align_bytes| { - const sp_global: Wasm.GlobalIndex = .stack_pointer; - // load stack pointer - w.writeByte(@intFromEnum(std.wasm.Opcode.global_get)) catch unreachable; - w.writeUleb128(@intFromEnum(sp_global)) catch unreachable; - // store stack pointer so we can restore it when we return from the function - w.writeByte(@intFromEnum(std.wasm.Opcode.local_tee)) catch unreachable; - w.writeUleb128(mir.prologue.sp_local) catch unreachable; - // get the total stack size - const aligned_stack: i32 = @intCast(stack_alignment.forward(mir.prologue.stack_size)); - w.writeByte(@intFromEnum(std.wasm.Opcode.i32_const)) catch unreachable; - w.writeSleb128(aligned_stack) catch unreachable; - // subtract it from the current stack pointer - w.writeByte(@intFromEnum(std.wasm.Opcode.i32_sub)) catch unreachable; - // Get negative stack alignment - const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1; - w.writeByte(@intFromEnum(std.wasm.Opcode.i32_const)) catch unreachable; - w.writeSleb128(neg_stack_align) catch unreachable; - // Bitwise-and the value to get the new stack pointer to ensure the - // pointers are aligned with the abi alignment. - w.writeByte(@intFromEnum(std.wasm.Opcode.i32_and)) catch unreachable; - // The bottom will be used to calculate all stack pointer offsets. - w.writeByte(@intFromEnum(std.wasm.Opcode.local_tee)) catch unreachable; - w.writeUleb128(mir.prologue.bottom_stack_local) catch unreachable; - // Store the current stack pointer value into the global stack pointer so other function calls will - // start from this value instead and not overwrite the current stack. - w.writeByte(@intFromEnum(std.wasm.Opcode.global_set)) catch unreachable; - w.writeUleb128(@intFromEnum(sp_global)) catch unreachable; - } - - code.items.len += w.end; - - var emit: Emit = .{ - .mir = mir.*, - .wasm = wasm, - .code = code, - }; - try emit.lowerToCode(); -} - -pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data: T, end: usize } { - const fields = std.meta.fields(T); - var i: usize = index; - var result: T = undefined; - inline for (fields) |field| { - @field(result, field.name) = switch (field.type) { - u32 => self.extra[i], - i32 => @bitCast(self.extra[i]), - Wasm.UavsObjIndex, - Wasm.UavsExeIndex, - InternPool.Nav.Index, - InternPool.Index, - => @enumFromInt(self.extra[i]), - else => |field_type| @compileError("Unsupported field type " ++ @typeName(field_type)), - }; - i += 1; - } - - return .{ .data = result, .end = i }; -} - -pub const JumpTable = struct { - /// Length of the jump table and the amount of entries it contains (includes default) - length: u32, -}; - -pub const Imm64 = struct { - msb: u32, - lsb: u32, - - pub fn init(full: u64) Imm64 { - return .{ - .msb = @truncate(full >> 32), - .lsb = @truncate(full), - }; - } - - pub fn toInt(i: Imm64) u64 { - return (@as(u64, i.msb) << 32) | @as(u64, i.lsb); - } -}; - -pub const Float64 = struct { - msb: u32, - lsb: u32, - - pub fn init(f: f64) Float64 { - const int: u64 = @bitCast(f); - return .{ - .msb = @truncate(int >> 32), - .lsb = @truncate(int), - }; - } - - pub fn toInt(f: Float64) u64 { - return (@as(u64, f.msb) << 32) | @as(u64, f.lsb); - } -}; - -pub const MemArg = struct { - offset: u32, - alignment: u32, -}; - -pub const UavRefOff = struct { - value: InternPool.Index, - offset: i32, -}; - -pub const NavRefOff = struct { - nav_index: InternPool.Nav.Index, - offset: i32, -}; - -/// Maps a source line with wasm bytecode -pub const DbgLineColumn = struct { - line: u32, - column: u32, -}; - -/// Tag names exactly match the corresponding symbol name. -pub const Intrinsic = enum(u32) { - __addhf3, - __addtf3, - __addxf3, - __ashlti3, - __ashrti3, - __bitreversedi2, - __bitreversesi2, - __bswapdi2, - __bswapsi2, - __ceilh, - __ceilx, - __cosh, - __cosx, - __divhf3, - __divtf3, - __divti3, - __divxf3, - __eqtf2, - __eqxf2, - __exp2h, - __exp2x, - __exph, - __expx, - __extenddftf2, - __extenddfxf2, - __extendhfsf2, - __extendhftf2, - __extendhfxf2, - __extendsftf2, - __extendsfxf2, - __extendxftf2, - __fabsh, - __fabsx, - __fixdfdi, - __fixdfsi, - __fixdfti, - __fixhfdi, - __fixhfsi, - __fixhfti, - __fixsfdi, - __fixsfsi, - __fixsfti, - __fixtfdi, - __fixtfsi, - __fixtfti, - __fixunsdfdi, - __fixunsdfsi, - __fixunsdfti, - __fixunshfdi, - __fixunshfsi, - __fixunshfti, - __fixunssfdi, - __fixunssfsi, - __fixunssfti, - __fixunstfdi, - __fixunstfsi, - __fixunstfti, - __fixunsxfdi, - __fixunsxfsi, - __fixunsxfti, - __fixxfdi, - __fixxfsi, - __fixxfti, - __floatdidf, - __floatdihf, - __floatdisf, - __floatditf, - __floatdixf, - __floatsidf, - __floatsihf, - __floatsisf, - __floatsitf, - __floatsixf, - __floattidf, - __floattihf, - __floattisf, - __floattitf, - __floattixf, - __floatundidf, - __floatundihf, - __floatundisf, - __floatunditf, - __floatundixf, - __floatunsidf, - __floatunsihf, - __floatunsisf, - __floatunsitf, - __floatunsixf, - __floatuntidf, - __floatuntihf, - __floatuntisf, - __floatuntitf, - __floatuntixf, - __floorh, - __floorx, - __fmah, - __fmax, - __fmaxh, - __fmaxx, - __fminh, - __fminx, - __fmodh, - __fmodx, - __getf2, - __gexf2, - __gttf2, - __gtxf2, - __letf2, - __lexf2, - __log10h, - __log10x, - __log2h, - __log2x, - __logh, - __logx, - __lshrti3, - __lttf2, - __ltxf2, - __modti3, - __mulhf3, - __mulodi4, - __muloti4, - __multf3, - __multi3, - __mulxf3, - __netf2, - __nexf2, - __roundh, - __roundx, - __sinh, - __sinx, - __sqrth, - __sqrtx, - __subhf3, - __subtf3, - __subxf3, - __tanh, - __tanx, - __trunch, - __truncsfhf2, - __trunctfdf2, - __trunctfhf2, - __trunctfsf2, - __trunctfxf2, - __truncx, - __truncxfdf2, - __truncxfhf2, - __truncxfsf2, - __udivti3, - __umodti3, - ceilq, - cos, - cosf, - cosq, - exp, - exp2, - exp2f, - exp2q, - expf, - expq, - fabsq, - floorq, - fma, - fmaf, - fmaq, - fmax, - fmaxf, - fmaxq, - fmin, - fminf, - fminq, - fmod, - fmodf, - fmodq, - log, - log10, - log10f, - log10q, - log2, - log2f, - log2q, - logf, - logq, - roundq, - sin, - sinf, - sinq, - sqrtq, - tan, - tanf, - tanq, - truncq, -}; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig deleted file mode 100644 index c62d54ccecf0964c94f99331a3c5cffc26082224..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/CodeGen.zig +++ /dev/null @@ -1,194725 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const codegen = @import("../../codegen.zig"); -const link = @import("../../link.zig"); -const log = std.log.scoped(.codegen); -const tracking_log = std.log.scoped(.tracking); -const verbose_tracking_log = std.log.scoped(.verbose_tracking); -const wip_mir_log = std.log.scoped(.wip_mir); -const Writer = std.Io.Writer; - -const Air = @import("../../Air.zig"); -const Allocator = std.mem.Allocator; -const Emit = @import("Emit.zig"); -const Lower = @import("Lower.zig"); -const Mir = @import("Mir.zig"); -const Zcu = @import("../../Zcu.zig"); -const Module = @import("../../Package/Module.zig"); -const InternPool = @import("../../InternPool.zig"); -const Type = @import("../../Type.zig"); -const Value = @import("../../Value.zig"); - -const abi = @import("abi.zig"); -const bits = @import("bits.zig"); -const encoder = @import("encoder.zig"); - -const Condition = bits.Condition; -const Immediate = bits.Immediate; -const Memory = bits.Memory; -const Register = bits.Register; -const RegisterManager = abi.RegisterManager; -const RegisterLock = RegisterManager.RegisterLock; -const FrameIndex = bits.FrameIndex; - -const InnerError = codegen.CodeGenError || error{OutOfRegisters}; - -pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { - return comptime &.initMany(&.{ - .scalarize_mul_sat, - .scalarize_div_floor, - .scalarize_mod, - .scalarize_add_with_overflow, - .scalarize_sub_with_overflow, - .scalarize_mul_with_overflow, - .scalarize_shl_with_overflow, - .scalarize_shr, - .scalarize_shr_exact, - .scalarize_shl, - .scalarize_shl_exact, - .scalarize_shl_sat, - .scalarize_bitcast, - .scalarize_ctz, - .scalarize_popcount, - .scalarize_byte_swap, - .scalarize_bit_reverse, - .scalarize_cmp_vector, - .scalarize_cmp_vector_optimized, - .scalarize_shuffle_one, - .scalarize_shuffle_two, - .scalarize_select, - - //.unsplat_shift_rhs, - .reduce_one_elem_to_bitcast, - .splat_one_elem_to_bitcast, - - .expand_intcast_safe, - .expand_int_from_float_safe, - .expand_int_from_float_optimized_safe, - .expand_add_safe, - .expand_sub_safe, - .expand_mul_safe, - - .expand_packed_load, - .expand_packed_store, - .expand_packed_struct_field_val, - .expand_packed_aggregate_init, - }); -} - -/// Set this to `false` to uncover Sema OPV bugs. -/// https://github.com/ziglang/zig/issues/22419 -const hack_around_sema_opv_bugs = true; - -const err_ret_trace_index: Air.Inst.Index = @enumFromInt(std.math.maxInt(u32)); - -gpa: Allocator, -pt: Zcu.PerThread, -air: Air, -liveness: Air.Liveness, -target: *const std.Target, -owner: union(enum) { - nav_index: InternPool.Nav.Index, - lazy_sym: link.File.LazySymbol, -}, -inline_func: InternPool.Index, -mod: *Module, -args: []MCValue, -va_info: union { - sysv: struct { - gp_count: u32, - fp_count: u32, - overflow_arg_area: bits.FrameAddr, - reg_save_area: bits.FrameAddr, - }, - win64: struct {}, -}, -ret_mcv: InstTracking, -err_ret_trace_reg: Register, -fn_type: Type, -src_loc: Zcu.LazySrcLoc, - -eflags_inst: ?Air.Inst.Index = null, - -/// MIR Instructions -mir_instructions: std.MultiArrayList(Mir.Inst) = .empty, -/// MIR extra data -mir_extra: std.ArrayListUnmanaged(u32) = .empty, -mir_string_bytes: std.ArrayListUnmanaged(u8) = .empty, -mir_strings: std.HashMapUnmanaged( - u32, - void, - std.hash_map.StringIndexContext, - std.hash_map.default_max_load_percentage, -) = .empty, -mir_locals: std.ArrayListUnmanaged(Mir.Local) = .empty, -mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, - -/// The value is an offset into the `Function` `code` from the beginning. -/// To perform the reloc, write 32-bit signed little-endian integer -/// which is a relative jump, based on the address following the reloc. -epilogue_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, - -reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, -inst_tracking: InstTrackingMap = .empty, - -// Key is the block instruction -blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, - -register_manager: RegisterManager = .{}, - -/// Generation of the current scope, increments by 1 for every entered scope. -scope_generation: u32 = 0, - -frame_allocs: std.MultiArrayList(FrameAlloc) = .empty, -free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty, -frame_locs: std.MultiArrayList(Mir.FrameLoc) = .empty, - -loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { - /// The state to restore before branching. - state: State, - /// The branch target. - target: Mir.Inst.Index, -}) = .empty, -loop_switches: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { - start: u31, - len: u11, - min: Value, - else_relocs: union(enum) { - @"unreachable", - forward: std.ArrayListUnmanaged(Mir.Inst.Index), - backward: Mir.Inst.Index, - }, -}) = .empty, - -next_temp_index: Temp.Index = @enumFromInt(0), -temp_type: [Temp.Index.max]Type = undefined, - -const MaskInfo = packed struct { - kind: enum(u1) { sign, all }, - inverted: bool = false, - scalar: Memory.Size, -}; - -pub const MCValue = union(enum) { - /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. - /// TODO Look into deleting this tag and using `dead` instead, since every use - /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. - none, - /// Control flow will not allow this value to be observed. - unreach, - /// No more references to this value remain. - /// The payload is the value of scope_generation at the point where the death occurred - dead: u32, - /// The value is undefined. - undef, - /// A pointer-sized integer that fits in a register. - /// If the type is a pointer, this is the pointer address in virtual address space. - immediate: u64, - /// The value resides in the EFLAGS register. - eflags: Condition, - /// The value is in a register. - register: Register, - /// The value is split across two registers. - register_pair: [2]Register, - /// The value is split across three registers. - register_triple: [3]Register, - /// The value is split across four registers. - register_quadruple: [4]Register, - /// The value is a constant offset from the value in a register. - register_offset: bits.RegisterOffset, - /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register. - register_overflow: struct { reg: Register, eflags: Condition }, - /// The value is a bool vector stored in a vector register with a different scalar type. - register_mask: struct { reg: Register, info: MaskInfo }, - /// The value is in memory at a hard-coded address. - /// If the type is a pointer, it means the pointer address is stored at this memory location. - memory: u64, - /// The value is in memory at a constant offset from the address in a register. - indirect: bits.RegisterOffset, - indirect_load_frame: bits.FrameAddr, - /// The value stored at an offset from a frame index - /// Payload is a frame address. - load_frame: bits.FrameAddr, - /// The address of an offset from a frame index - /// Payload is a frame address. - lea_frame: bits.FrameAddr, - load_nav: InternPool.Nav.Index, - lea_nav: InternPool.Nav.Index, - load_uav: InternPool.Key.Ptr.BaseAddr.Uav, - lea_uav: InternPool.Key.Ptr.BaseAddr.Uav, - load_lazy_sym: link.File.LazySymbol, - lea_lazy_sym: link.File.LazySymbol, - load_extern_func: Mir.NullTerminatedString, - lea_extern_func: Mir.NullTerminatedString, - /// Supports integer_per_element abi - elementwise_args: packed struct { regs: u3, frame_off: i29, frame_index: FrameIndex }, - /// This indicates that we have already allocated a frame index for this instruction, - /// but it has not been spilled there yet in the current control flow. - /// Payload is a frame index. - reserved_frame: FrameIndex, - air_ref: Air.Inst.Ref, - - fn isModifiable(mcv: MCValue) bool { - return switch (mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .register_offset, - .register_mask, - .eflags, - .register_overflow, - .indirect_load_frame, - .lea_frame, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .lea_extern_func, - .load_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => false, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .memory, - .indirect, - .load_nav, - => true, - .load_frame => |frame_addr| !frame_addr.index.isNamed(), - }; - } - - // hack around linker relocation bugs - fn isBase(mcv: MCValue) bool { - return switch (mcv) { - .memory, .indirect, .load_frame => true, - else => false, - }; - } - - fn isMemory(mcv: MCValue) bool { - return switch (mcv) { - .memory, - .indirect, - .load_frame, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => true, - else => false, - }; - } - - fn isImmediate(mcv: MCValue) bool { - return switch (mcv) { - .immediate => true, - else => false, - }; - } - - fn isRegister(mcv: MCValue) bool { - return switch (mcv) { - .register => true, - .register_offset => |reg_off| return reg_off.off == 0, - else => false, - }; - } - - fn isRegisterOffset(mcv: MCValue) bool { - return switch (mcv) { - .register, .register_offset => true, - else => false, - }; - } - - fn getReg(mcv: MCValue) ?Register { - return switch (mcv) { - .register => |reg| reg, - .register_offset, .indirect => |ro| ro.reg, - .register_overflow => |ro| ro.reg, - .register_mask => |rm| rm.reg, - else => null, - }; - } - - fn getRegs(mcv: *const MCValue) []const Register { - return switch (mcv.*) { - .register => |*reg| reg[0..1], - inline .register_pair, - .register_triple, - .register_quadruple, - => |*regs| regs, - inline .register_offset, - .indirect, - .register_overflow, - .register_mask, - => |*pl| (&pl.reg)[0..1], - else => &.{}, - }; - } - - fn getCondition(mcv: MCValue) ?Condition { - return switch (mcv) { - .eflags => |cc| cc, - .register_overflow => |reg_ov| reg_ov.eflags, - else => null, - }; - } - - fn isAddress(mcv: MCValue) bool { - return switch (mcv) { - .immediate, .register, .register_offset, .lea_frame => true, - else => false, - }; - } - - fn address(mcv: MCValue) MCValue { - return switch (mcv) { - .none => .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .register_overflow, - .register_mask, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // not in memory - .memory => |addr| .{ .immediate = addr }, - .indirect => |reg_off| switch (reg_off.off) { - 0 => .{ .register = reg_off.reg }, - else => .{ .register_offset = reg_off }, - }, - .indirect_load_frame => |frame_addr| .{ .load_frame = frame_addr }, - .load_frame => |frame_addr| .{ .lea_frame = frame_addr }, - .load_nav => |nav| .{ .lea_nav = nav }, - .load_uav => |uav| .{ .lea_uav = uav }, - .load_lazy_sym => |lazy_sym| .{ .lea_lazy_sym = lazy_sym }, - .load_extern_func => |extern_func| .{ .lea_extern_func = extern_func }, - }; - } - - fn deref(mcv: MCValue) MCValue { - return switch (mcv) { - .none => .none, - .unreach, - .dead, - .undef, - .eflags, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - .register_mask, - .memory, - .indirect, - .indirect_load_frame, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // not dereferenceable - .immediate => |addr| .{ .memory = addr }, - .register => |reg| .{ .indirect = .{ .reg = reg } }, - .register_offset => |reg_off| .{ .indirect = reg_off }, - .load_frame => |frame_addr| .{ .indirect_load_frame = frame_addr }, - .lea_frame => |frame_addr| .{ .load_frame = frame_addr }, - .lea_nav => |nav| .{ .load_nav = nav }, - .lea_uav => |uav| .{ .load_uav = uav }, - .lea_lazy_sym => |lazy_sym| .{ .load_lazy_sym = lazy_sym }, - .lea_extern_func => |extern_func| .{ .load_extern_func = extern_func }, - }; - } - - fn offset(mcv: MCValue, off: i32) MCValue { - return switch (mcv) { - .none => .none, - .unreach, - .dead, - .undef, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // not valid - .eflags, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - .register_mask, - .memory, - .indirect, - .indirect_load_frame, - .load_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => switch (off) { - 0 => mcv, - else => unreachable, // not offsettable - }, - .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) }, - .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } }, - .register_offset => |reg_off| .{ - .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off }, - }, - .lea_frame => |frame_addr| .{ - .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, - }, - }; - } - - fn mem(mcv: MCValue, function: *CodeGen, mod_rm: Memory.Mod.Rm) !Memory { - return switch (mcv) { - .none => .{ .mod = .{ .rm = mod_rm } }, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .register_overflow, - .register_mask, - .indirect_load_frame, - .lea_frame, - .elementwise_args, - .reserved_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => unreachable, - .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = mod_rm.size, - .index = mod_rm.index, - .scale = mod_rm.scale, - .disp = small_addr + mod_rm.disp, - } }, - } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } }, - .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg.toSize(.ptr, function.target) }, - .mod = .{ .rm = .{ - .size = mod_rm.size, - .index = mod_rm.index, - .scale = mod_rm.scale, - .disp = reg_off.off + mod_rm.disp, - } }, - }, - .load_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = mod_rm.size, - .index = mod_rm.index, - .scale = mod_rm.scale, - .disp = frame_addr.off + mod_rm.disp, - } }, - }, - .load_nav => |nav| .{ .base = .{ .nav = nav }, .mod = .{ .rm = mod_rm } }, - .load_uav => |uav| .{ .base = .{ .uav = uav }, .mod = .{ .rm = mod_rm } }, - .load_lazy_sym => |lazy_sym| .{ .base = .{ .lazy_sym = lazy_sym }, .mod = .{ .rm = mod_rm } }, - .load_extern_func => |extern_func| .{ .base = .{ .extern_func = extern_func }, .mod = .{ .rm = mod_rm } }, - .air_ref => |ref| (try function.resolveInst(ref)).mem(function, mod_rm), - }; - } - - pub fn format(mcv: MCValue, w: *Writer) Writer.Error!void { - switch (mcv) { - .none, .unreach, .dead, .undef => try w.print("({s})", .{@tagName(mcv)}), - .immediate => |pl| try w.print("0x{x}", .{pl}), - .memory => |pl| try w.print("[ds:0x{x}]", .{pl}), - inline .eflags, .register => |pl| try w.print("{s}", .{@tagName(pl)}), - .register_pair => |pl| try w.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), - .register_triple => |pl| try w.print("{s}:{s}:{s}", .{ - @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]), - }), - .register_quadruple => |pl| try w.print("{s}:{s}:{s}:{s}", .{ - @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]), - }), - .register_offset => |pl| try w.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }), - .register_overflow => |pl| try w.print("{s}:{s}", .{ - @tagName(pl.eflags), - @tagName(pl.reg), - }), - .register_mask => |pl| try w.print("mask({s},{f}):{c}{s}", .{ - @tagName(pl.info.kind), - pl.info.scalar, - @as(u8, if (pl.info.inverted) '!' else ' '), - @tagName(pl.reg), - }), - .indirect => |pl| try w.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), - .indirect_load_frame => |pl| try w.print("[[{f} + 0x{x}]]", .{ pl.index, pl.off }), - .load_frame => |pl| try w.print("[{f} + 0x{x}]", .{ pl.index, pl.off }), - .lea_frame => |pl| try w.print("{f} + 0x{x}", .{ pl.index, pl.off }), - .load_nav => |pl| try w.print("[nav:{d}]", .{@intFromEnum(pl)}), - .lea_nav => |pl| try w.print("nav:{d}", .{@intFromEnum(pl)}), - .load_uav => |pl| try w.print("[uav:{d}]", .{@intFromEnum(pl.val)}), - .lea_uav => |pl| try w.print("uav:{d}", .{@intFromEnum(pl.val)}), - .load_lazy_sym => |pl| try w.print("[lazy:{s}:{d}]", .{ @tagName(pl.kind), @intFromEnum(pl.ty) }), - .lea_lazy_sym => |pl| try w.print("lazy:{s}:{d}", .{ @tagName(pl.kind), @intFromEnum(pl.ty) }), - .load_extern_func => |pl| try w.print("[extern:{d}]", .{@intFromEnum(pl)}), - .lea_extern_func => |pl| try w.print("extern:{d}", .{@intFromEnum(pl)}), - .elementwise_args => |pl| try w.print("elementwise:{d}:[{f} + 0x{x}]", .{ - pl.regs, pl.frame_index, pl.frame_off, - }), - .reserved_frame => |pl| try w.print("(dead:{f})", .{pl}), - .air_ref => |pl| try w.print("(air:0x{x})", .{@intFromEnum(pl)}), - } - } -}; - -const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking); -const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking); -const InstTracking = struct { - long: MCValue, - short: MCValue, - - fn init(result: MCValue) InstTracking { - return .{ .long = switch (result) { - .none, - .unreach, - .undef, - .immediate, - .memory, - .indirect_load_frame, - .load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => result, - .dead, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .eflags, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .register_overflow, - .register_mask, - .indirect, - => .none, - }, .short = result }; - } - - fn getReg(self: InstTracking) ?Register { - return self.short.getReg(); - } - - fn getRegs(self: *const InstTracking) []const Register { - return self.short.getRegs(); - } - - fn getCondition(self: InstTracking) ?Condition { - return self.short.getCondition(); - } - - fn spill(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { - if (std.meta.eql(self.long, self.short)) return; // Already spilled - // Allocate or reuse frame index - switch (self.long) { - .none => self.long = try cg.allocRegOrMem(inst, false), - .load_frame => {}, - .lea_frame => return, - .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, - else => unreachable, - } - tracking_log.debug("spill {f} from {f} to {f}", .{ inst, self.short, self.long }); - try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{}); - for (self.short.getRegs()) |reg| if (reg.isClass(.x87)) try cg.asmRegister(.{ .f_, .free }, reg); - } - - fn reuseFrame(self: *InstTracking) void { - self.* = .init(switch (self.long) { - .none => switch (self.short) { - .dead => .none, - else => |short| short, - }, - .reserved_frame => |index| .{ .load_frame = .{ .index = index } }, - else => |long| long, - }); - } - - fn trackSpill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void { - switch (self.short) { - .register => |reg| function.register_manager.freeReg(reg), - inline .register_pair, - .register_triple, - .register_quadruple, - => |regs| for (regs) |reg| function.register_manager.freeReg(reg), - .register_offset, .indirect => |reg_off| function.register_manager.freeReg(reg_off.reg), - .register_overflow => |reg_ov| { - function.register_manager.freeReg(reg_ov.reg); - function.eflags_inst = null; - }, - .register_mask => |reg_mask| function.register_manager.freeReg(reg_mask.reg), - .eflags => function.eflags_inst = null, - else => {}, // TODO process stack allocation death - } - self.reuseFrame(); - tracking_log.debug("{f} => {f} (spilled)", .{ inst, self.* }); - } - - fn verifyMaterialize(self: InstTracking, target: InstTracking) void { - switch (self.long) { - .none, - .load_frame, - .reserved_frame, - => switch (target.long) { - .none, - .load_frame, - .reserved_frame, - => {}, - else => unreachable, - }, - .unreach, - .undef, - .immediate, - .memory, - .indirect_load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => assert(std.meta.eql(self.long, target.long)), - .dead, - .eflags, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .register_overflow, - .register_mask, - .indirect, - .elementwise_args, - .air_ref, - => unreachable, - } - } - - fn materialize( - self: *InstTracking, - function: *CodeGen, - inst: Air.Inst.Index, - target: InstTracking, - ) !void { - self.verifyMaterialize(target); - try self.materializeUnsafe(function, inst, target); - } - - fn materializeUnsafe( - self: InstTracking, - function: *CodeGen, - inst: Air.Inst.Index, - target: InstTracking, - ) !void { - const ty = function.typeOfIndex(inst); - if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame) - try function.genCopy(ty, target.long, self.short, .{}); - try function.genCopy(ty, target.short, self.short, .{}); - } - - fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void { - self.verifyMaterialize(target); - // Don't clobber reserved frame indices - self.long = if (target.long == .none) switch (self.long) { - .load_frame => |addr| .{ .reserved_frame = addr.index }, - .reserved_frame => self.long, - else => target.long, - } else target.long; - self.short = target.short; - tracking_log.debug("{f} => {f} (materialize)", .{ inst, self.* }); - } - - fn resurrect(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index, scope_generation: u32) !void { - switch (self.short) { - .dead => |die_generation| if (die_generation >= scope_generation) { - self.reuseFrame(); - try function.getValue(self.short, inst); - tracking_log.debug("{f} => {f} (resurrect)", .{ inst, self.* }); - }, - else => {}, - } - } - - fn die(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index, comptime opts: FreeOptions) !void { - if (self.short == .dead) return; - try function.freeValue(self.short, opts); - if (self.long == .none) self.long = self.short; - self.short = .{ .dead = function.scope_generation }; - tracking_log.debug("{f} => {f} (death)", .{ inst, self.* }); - } - - fn reuse( - self: *InstTracking, - function: *CodeGen, - new_inst: ?Air.Inst.Index, - old_inst: Air.Inst.Index, - ) void { - self.short = .{ .dead = function.scope_generation }; - tracking_log.debug("{?f} => {f} (reuse {f})", .{ new_inst, self.*, old_inst }); - } - - fn liveOut(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) void { - for (self.getRegs()) |reg| { - if (function.register_manager.isRegFree(reg)) { - tracking_log.debug("{f} => {f} (live-out)", .{ inst, self.* }); - continue; - } - - const index = RegisterManager.indexOfRegIntoTracked(reg).?; - const tracked_inst = function.register_manager.registers[index]; - const tracking = function.getResolvedInstValue(tracked_inst); - - // Disable death. - var found_reg = false; - var remaining_reg: Register = .none; - for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) { - assert(!found_reg); - found_reg = true; - } else { - assert(remaining_reg == .none); - remaining_reg = tracked_reg; - }; - assert(found_reg); - if (tracking.long == .none) tracking.long = tracking.short; - tracking.short = switch (remaining_reg) { - .none => .{ .dead = function.scope_generation }, - else => .{ .register = remaining_reg }, - }; - - // Perform side-effects of freeValue manually. - function.register_manager.freeReg(reg); - - tracking_log.debug("{f} => {f} (live-out {f})", .{ inst, self.*, tracked_inst }); - } - } - - pub fn format(tracking: InstTracking, bw: *Writer) Writer.Error!void { - if (!std.meta.eql(tracking.long, tracking.short)) try bw.print("|{f}| ", .{tracking.long}); - try bw.print("{f}", .{tracking.short}); - } -}; - -const FrameAlloc = struct { - abi_size: u31, - spill_pad: u3, - abi_align: InternPool.Alignment, - ref_count: u16, - - fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc { - return .{ - .abi_size = @intCast(alloc_abi.size), - .spill_pad = alloc_abi.pad, - .abi_align = alloc_abi.alignment, - .ref_count = 0, - }; - } - fn initType(ty: Type, zcu: *Zcu) FrameAlloc { - return init(.{ - .size = ty.abiSize(zcu), - .alignment = ty.abiAlignment(zcu), - }); - } - fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc { - const abi_size = ty.abiSize(zcu); - const spill_size = if (abi_size < 8) - std.math.ceilPowerOfTwoAssert(u64, abi_size) - else - std.mem.alignForward(u64, abi_size, 8); - return init(.{ - .size = spill_size, - .pad = @intCast(spill_size - abi_size), - .alignment = ty.abiAlignment(zcu).maxStrict( - .fromNonzeroByteUnits(@min(spill_size, 8)), - ), - }); - } -}; - -const StackAllocation = struct { - inst: ?Air.Inst.Index, - /// TODO do we need size? should be determined by inst.ty.abiSize(zcu) - size: u32, -}; - -const BlockData = struct { - relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, - state: State, - - fn deinit(self: *BlockData, gpa: Allocator) void { - self.relocs.deinit(gpa); - self.* = undefined; - } -}; - -const CodeGen = @This(); - -pub fn generate( - bin_file: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - air: *const Air, - liveness: *const ?Air.Liveness, -) codegen.CodeGenError!Mir { - _ = bin_file; - const zcu = pt.zcu; - const gpa = zcu.gpa; - const ip = &zcu.intern_pool; - const func = zcu.funcInfo(func_index); - const func_zir = func.zir_body_inst.resolveFull(ip).?; - const file = zcu.fileByIndex(func_zir.file); - const fn_type: Type = .fromInterned(func.ty); - const mod = file.mod.?; - - var function: CodeGen = .{ - .gpa = gpa, - .pt = pt, - .air = air.*, - .liveness = liveness.*.?, - .target = &mod.resolved_target.result, - .mod = mod, - .owner = .{ .nav_index = func.owner_nav }, - .inline_func = func_index, - .args = undefined, // populated after `resolveCallingConventionValues` - .va_info = undefined, // populated after `resolveCallingConventionValues` - .ret_mcv = undefined, // populated after `resolveCallingConventionValues` - .err_ret_trace_reg = undefined, // populated after `resolveCallingConventionValues` - .fn_type = fn_type, - .src_loc = src_loc, - }; - defer { - function.frame_allocs.deinit(gpa); - function.free_frame_indices.deinit(gpa); - function.frame_locs.deinit(gpa); - function.loops.deinit(gpa); - function.loop_switches.deinit(gpa); - var block_it = function.blocks.valueIterator(); - while (block_it.next()) |block| block.deinit(gpa); - function.blocks.deinit(gpa); - function.inst_tracking.deinit(gpa); - function.epilogue_relocs.deinit(gpa); - function.mir_instructions.deinit(gpa); - function.mir_string_bytes.deinit(gpa); - function.mir_strings.deinit(gpa); - function.mir_locals.deinit(gpa); - function.mir_extra.deinit(gpa); - function.mir_table.deinit(gpa); - } - try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max); - for (0..Temp.Index.max) |temp_index| { - const temp: Temp.Index = @enumFromInt(temp_index); - function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none)); - } - - wip_mir_log.debug("{f}:", .{fmtNav(func.owner_nav, ip)}); - - try function.frame_allocs.resize(gpa, FrameIndex.named_count); - function.frame_allocs.set( - @intFromEnum(FrameIndex.stack_frame), - .init(.{ .size = 0, .alignment = .@"1" }), - ); - function.frame_allocs.set( - @intFromEnum(FrameIndex.call_frame), - .init(.{ .size = 0, .alignment = .@"1" }), - ); - - const fn_info = zcu.typeToFunc(fn_type).?; - var call_info = function.resolveCallingConventionValues(fn_info, &.{}, .args_frame) catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - else => |e| return e, - }; - defer call_info.deinit(&function); - - function.args = call_info.args; - function.ret_mcv = call_info.return_value; - function.err_ret_trace_reg = call_info.err_ret_trace_reg; - function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), .init(.{ - .size = Type.usize.abiSize(zcu), - .alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align), - })); - function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), .init(.{ - .size = Type.usize.abiSize(zcu), - .alignment = call_info.stack_align.min( - .fromNonzeroByteUnits(function.target.stackAlignment()), - ), - })); - function.frame_allocs.set( - @intFromEnum(FrameIndex.args_frame), - .init(.{ - .size = call_info.stack_byte_count, - .alignment = call_info.stack_align, - }), - ); - function.va_info = switch (fn_info.cc) { - else => undefined, - .x86_64_sysv => .{ .sysv = .{ - .gp_count = call_info.gp_count, - .fp_count = call_info.fp_count, - .overflow_arg_area = .{ .index = .args_frame, .off = call_info.stack_byte_count }, - .reg_save_area = undefined, - } }, - .x86_64_win => .{ .win64 = .{} }, - }; - if (call_info.err_ret_trace_reg != .none) { - function.register_manager.getRegAssumeFree(call_info.err_ret_trace_reg, err_ret_trace_index); - try function.inst_tracking.putNoClobber( - gpa, - err_ret_trace_index, - .init(.{ .register = call_info.err_ret_trace_reg }), - ); - } - - function.gen(&file.zir.?, func_zir.inst, func.comptime_args, call_info.air_arg_count) catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), - else => |e| return e, - }; - - // Drop them off at the rbrace. - if (!mod.strip) _ = try function.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_line_line_column, - .data = .{ .line_column = .{ - .line = func.rbrace_line, - .column = func.rbrace_column, - } }, - }); - - var mir: Mir = .{ - .instructions = .empty, - .extra = &.{}, - .string_bytes = &.{}, - .locals = &.{}, - .table = &.{}, - .frame_locs = .empty, - }; - errdefer mir.deinit(gpa); - mir.instructions = function.mir_instructions.toOwnedSlice(); - mir.extra = try function.mir_extra.toOwnedSlice(gpa); - mir.string_bytes = try function.mir_string_bytes.toOwnedSlice(gpa); - mir.locals = try function.mir_locals.toOwnedSlice(gpa); - mir.table = try function.mir_table.toOwnedSlice(gpa); - mir.frame_locs = function.frame_locs.toOwnedSlice(); - return mir; -} - -pub fn getTmpMir(cg: *CodeGen) Mir { - return .{ - .instructions = cg.mir_instructions.slice(), - .extra = cg.mir_extra.items, - .string_bytes = cg.mir_string_bytes.items, - .locals = cg.mir_locals.items, - .table = cg.mir_table.items, - .frame_locs = cg.frame_locs.slice(), - }; -} - -pub fn generateLazy( - bin_file: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - lazy_sym: link.File.LazySymbol, - atom_index: u32, - w: *std.Io.Writer, - debug_output: link.File.DebugInfoOutput, -) codegen.CodeGenError!void { - const gpa = pt.zcu.gpa; - // This function is for generating global code, so we use the root module. - const mod = pt.zcu.comp.root_mod; - var function: CodeGen = .{ - .gpa = gpa, - .pt = pt, - .air = undefined, - .liveness = undefined, - .target = &mod.resolved_target.result, - .mod = mod, - .owner = .{ .lazy_sym = lazy_sym }, - .inline_func = undefined, - .args = undefined, - .va_info = undefined, - .ret_mcv = undefined, - .err_ret_trace_reg = undefined, - .fn_type = undefined, - .src_loc = src_loc, - }; - defer { - function.inst_tracking.deinit(gpa); - function.mir_instructions.deinit(gpa); - function.mir_string_bytes.deinit(gpa); - function.mir_strings.deinit(gpa); - function.mir_locals.deinit(gpa); - function.mir_extra.deinit(gpa); - function.mir_table.deinit(gpa); - } - try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max); - for (0..Temp.Index.max) |temp_index| { - const temp: Temp.Index = @enumFromInt(temp_index); - function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none)); - } - - function.genLazy(lazy_sym) catch |err| switch (err) { - error.CodegenFail => return error.CodegenFail, - error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), - else => |e| return e, - }; - - try function.getTmpMir().emitLazy(bin_file, pt, src_loc, lazy_sym, atom_index, w, debug_output); -} - -const FormatNavData = struct { - ip: *const InternPool, - nav_index: InternPool.Nav.Index, -}; -fn formatNav(data: FormatNavData, w: *Writer) Writer.Error!void { - try w.print("{f}", .{data.ip.getNav(data.nav_index).fqn.fmt(data.ip)}); -} -fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Alt(FormatNavData, formatNav) { - return .{ .data = .{ - .ip = ip, - .nav_index = nav_index, - } }; -} - -const FormatAirData = struct { - self: *CodeGen, - inst: Air.Inst.Index, -}; -fn formatAir(data: FormatAirData, w: *Writer) Writer.Error!void { - data.self.air.writeInst(w, data.inst, data.self.pt, data.self.liveness); -} -fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Alt(FormatAirData, formatAir) { - return .{ .data = .{ .self = self, .inst = inst } }; -} - -const FormatWipMirData = struct { - self: *CodeGen, - inst: Mir.Inst.Index, -}; -fn formatWipMir(data: FormatWipMirData, w: *Writer) Writer.Error!void { - var lower: Lower = .{ - .target = data.self.target, - .allocator = data.self.gpa, - .mir = data.self.getTmpMir(), - .cc = .auto, - .src_loc = data.self.src_loc, - }; - var first = true; - for ((lower.lowerMir(data.inst) catch |err| switch (err) { - error.LowerFail => { - defer { - lower.err_msg.?.deinit(data.self.gpa); - lower.err_msg = null; - } - try w.writeAll(lower.err_msg.?.msg); - return; - }, - else => |e| { - try w.writeAll(@errorName(e)); - return; - }, - }).insts) |lowered_inst| { - if (!first) try w.writeAll("\ndebug(wip_mir): "); - try w.print(" | {f}", .{lowered_inst}); - first = false; - } - if (first) { - const ip = &data.self.pt.zcu.intern_pool; - const mir_inst = lower.mir.instructions.get(data.inst); - try w.print(" | .{s}", .{@tagName(mir_inst.ops)}); - switch (mir_inst.ops) { - else => unreachable, - .pseudo_dbg_prologue_end_none, - .pseudo_dbg_epilogue_begin_none, - .pseudo_dbg_enter_block_none, - .pseudo_dbg_leave_block_none, - .pseudo_dbg_arg_none, - .pseudo_dbg_var_args_none, - .pseudo_dbg_var_none, - .pseudo_dead_none, - => {}, - .pseudo_dbg_line_stmt_line_column, .pseudo_dbg_line_line_column => try w.print( - " {[line]d}, {[column]d}", - mir_inst.data.line_column, - ), - .pseudo_dbg_enter_inline_func, .pseudo_dbg_leave_inline_func => try w.print(" {f}", .{ - ip.getNav(ip.indexToKey(mir_inst.data.ip_index).func.owner_nav).name.fmt(ip), - }), - .pseudo_dbg_arg_i_s, .pseudo_dbg_var_i_s => try w.print(" {d}", .{ - @as(i32, @bitCast(mir_inst.data.i.i)), - }), - .pseudo_dbg_arg_i_u, .pseudo_dbg_var_i_u => try w.print(" {d}", .{ - mir_inst.data.i.i, - }), - .pseudo_dbg_arg_i_64, .pseudo_dbg_var_i_64 => try w.print(" {d}", .{ - mir_inst.data.i64, - }), - .pseudo_dbg_arg_ro, .pseudo_dbg_var_ro => { - const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ - .base = .{ .reg = mir_inst.data.ro.reg }, - .disp = mir_inst.data.ro.off, - }) }; - try w.print(" {f}", .{mem_op.fmt(.m)}); - }, - .pseudo_dbg_arg_fa, .pseudo_dbg_var_fa => { - const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ - .base = .{ .frame = mir_inst.data.fa.index }, - .disp = mir_inst.data.fa.off, - }) }; - try w.print(" {f}", .{mem_op.fmt(.m)}); - }, - .pseudo_dbg_arg_m, .pseudo_dbg_var_m => { - const mem_op: encoder.Instruction.Operand = .{ - .mem = lower.mir.extraData(Mir.Memory, mir_inst.data.x.payload).data.decode(), - }; - try w.print(" {f}", .{mem_op.fmt(.m)}); - }, - .pseudo_dbg_arg_val, .pseudo_dbg_var_val => try w.print(" {f}", .{ - Value.fromInterned(mir_inst.data.ip_index).fmtValue(data.self.pt), - }), - } - } -} -fn fmtWipMir(self: *CodeGen, inst: Mir.Inst.Index) std.fmt.Alt(FormatWipMirData, formatWipMir) { - return .{ .data = .{ .self = self, .inst = inst } }; -} - -const FormatTrackingData = struct { - self: *CodeGen, -}; -fn formatTracking(data: FormatTrackingData, w: *Writer) Writer.Error!void { - var it = data.self.inst_tracking.iterator(); - while (it.next()) |entry| try w.print("\n{f} = {f}", .{ entry.key_ptr.*, entry.value_ptr.* }); -} -fn fmtTracking(self: *CodeGen) std.fmt.Alt(FormatTrackingData, formatTracking) { - return .{ .data = .{ .self = self } }; -} - -fn addInst(self: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { - const gpa = self.gpa; - try self.mir_instructions.ensureUnusedCapacity(gpa, 1); - const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len); - self.mir_instructions.appendAssumeCapacity(inst); - if (inst.ops != .pseudo_dead_none) wip_mir_log.debug("{f}", .{self.fmtWipMir(result_index)}); - return result_index; -} - -fn addExtra(self: *CodeGen, extra: anytype) Allocator.Error!u32 { - const fields = std.meta.fields(@TypeOf(extra)); - try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len); - return self.addExtraAssumeCapacity(extra); -} - -fn addExtraAssumeCapacity(self: *CodeGen, extra: anytype) u32 { - const fields = std.meta.fields(@TypeOf(extra)); - const result: u32 = @intCast(self.mir_extra.items.len); - inline for (fields) |field| { - self.mir_extra.appendAssumeCapacity(switch (field.type) { - u32 => @field(extra, field.name), - i32, Mir.Memory.Info => @bitCast(@field(extra, field.name)), - FrameIndex => @intFromEnum(@field(extra, field.name)), - else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)), - }); - } - return result; -} - -fn addString(cg: *CodeGen, string: []const u8) Allocator.Error!Mir.NullTerminatedString { - try cg.mir_string_bytes.ensureUnusedCapacity(cg.gpa, string.len + 1); - try cg.mir_strings.ensureUnusedCapacityContext(cg.gpa, 1, .{ .bytes = &cg.mir_string_bytes }); - - const mir_string_gop = cg.mir_strings.getOrPutAssumeCapacityAdapted( - string, - std.hash_map.StringIndexAdapter{ .bytes = &cg.mir_string_bytes }, - ); - if (!mir_string_gop.found_existing) { - mir_string_gop.key_ptr.* = @intCast(cg.mir_string_bytes.items.len); - cg.mir_string_bytes.appendSliceAssumeCapacity(string); - cg.mir_string_bytes.appendAssumeCapacity(0); - } - return @enumFromInt(mir_string_gop.key_ptr.*); -} - -fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void { - return switch (ops[0]) { - .none => self.asmOpOnly(tag), - .reg => |reg0| switch (ops[1]) { - .none => self.asmRegister(tag, reg0), - .reg => |reg1| switch (ops[2]) { - .none => self.asmRegisterRegister(tag, reg0, reg1), - .reg => |reg2| switch (ops[3]) { - .none => self.asmRegisterRegisterRegister(tag, reg0, reg1, reg2), - .reg => |reg3| self.asmRegisterRegisterRegisterRegister(tag, reg0, reg1, reg2, reg3), - .imm => |imm3| self.asmRegisterRegisterRegisterImmediate(tag, reg0, reg1, reg2, imm3), - else => error.InvalidInstruction, - }, - .mem => |mem2| switch (ops[3]) { - .none => self.asmRegisterRegisterMemory(tag, reg0, reg1, mem2), - .reg => |reg3| self.asmRegisterRegisterMemoryRegister(tag, reg0, reg1, mem2, reg3), - .imm => |imm3| self.asmRegisterRegisterMemoryImmediate(tag, reg0, reg1, mem2, imm3), - else => error.InvalidInstruction, - }, - .imm => |imm2| switch (ops[3]) { - .none => self.asmRegisterRegisterImmediate(tag, reg0, reg1, imm2), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .mem => |mem1| switch (ops[2]) { - .none => self.asmRegisterMemory(tag, reg0, mem1), - .reg => |reg2| switch (ops[3]) { - .none => self.asmRegisterMemoryRegister(tag, reg0, mem1, reg2), - else => error.InvalidInstruction, - }, - .imm => |imm2| switch (ops[3]) { - .none => self.asmRegisterMemoryImmediate(tag, reg0, mem1, imm2), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .imm => |imm1| switch (ops[2]) { - .none => self.asmRegisterImmediate(tag, reg0, imm1), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .mem => |mem0| switch (ops[1]) { - .none => self.asmMemory(tag, mem0), - .reg => |reg1| switch (ops[2]) { - .none => self.asmMemoryRegister(tag, mem0, reg1), - .reg => |reg2| switch (ops[3]) { - .none => self.asmMemoryRegisterRegister(tag, mem0, reg1, reg2), - else => error.InvalidInstruction, - }, - .imm => |imm2| switch (ops[3]) { - .none => self.asmMemoryRegisterImmediate(tag, mem0, reg1, imm2), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .imm => |imm1| switch (ops[2]) { - .none => self.asmMemoryImmediate(tag, mem0, imm1), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .imm => |imm0| switch (ops[1]) { - .none => self.asmImmediate(tag, imm0), - .reg => |reg1| switch (ops[2]) { - .none => self.asmImmediateRegister(tag, imm0, reg1), - else => error.InvalidInstruction, - }, - .imm => |imm1| switch (ops[2]) { - .none => self.asmImmediateImmediate(tag, imm0, imm1), - else => error.InvalidInstruction, - }, - else => error.InvalidInstruction, - }, - .inst => |inst0| switch (ops[1]) { - .none => self.asmReloc(tag, inst0), - else => error.InvalidInstruction, - }, - }; -} - -/// A `cc` of `.z_and_np` clobbers `reg2`! -fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void { - if (self.hasFeature(.cmov)) _ = try self.addInst(.{ - .tag = switch (cc) { - else => .cmov, - .z_and_np, .nz_or_p => .pseudo, - }, - .ops = switch (cc) { - else => .rr, - .z_and_np => .pseudo_cmov_z_and_np_rr, - .nz_or_p => .pseudo_cmov_nz_or_p_rr, - }, - .data = .{ .rr = .{ - .fixes = switch (cc) { - else => .fromCond(cc), - .z_and_np, .nz_or_p => ._, - }, - .r1 = reg1, - .r2 = reg2, - } }, - }) else { - const reloc = try self.asmJccReloc(cc.negate(), undefined); - try self.asmRegisterRegister(.{ ._, .mov }, reg1, reg2); - self.performReloc(reloc); - } -} - -/// A `cc` of `.z_and_np` is not supported by this encoding! -fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void { - if (self.hasFeature(.cmov)) _ = try self.addInst(.{ - .tag = switch (cc) { - else => .cmov, - .z_and_np => unreachable, - .nz_or_p => .pseudo, - }, - .ops = switch (cc) { - else => .rm, - .z_and_np => unreachable, - .nz_or_p => .pseudo_cmov_nz_or_p_rm, - }, - .data = .{ .rx = .{ - .fixes = switch (cc) { - else => .fromCond(cc), - .z_and_np => unreachable, - .nz_or_p => ._, - }, - .r1 = reg, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }) else { - const reloc = try self.asmJccReloc(cc.negate(), undefined); - try self.asmRegisterMemory(.{ ._, .mov }, reg, m); - self.performReloc(reloc); - } -} - -fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void { - _ = try self.addInst(.{ - .tag = switch (cc) { - else => .set, - .z_and_np, .nz_or_p => .pseudo, - }, - .ops = switch (cc) { - else => .r, - .z_and_np => .pseudo_set_z_and_np_r, - .nz_or_p => .pseudo_set_nz_or_p_r, - }, - .data = switch (cc) { - else => .{ .r = .{ - .fixes = .fromCond(cc), - .r1 = reg, - } }, - .z_and_np, .nz_or_p => .{ .rr = .{ - .r1 = reg, - .r2 = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to8(), - } }, - }, - }); -} - -fn asmSetccMemory(self: *CodeGen, cc: Condition, m: Memory) !void { - const payload = try self.addExtra(Mir.Memory.encode(m)); - _ = try self.addInst(.{ - .tag = switch (cc) { - else => .set, - .z_and_np, .nz_or_p => .pseudo, - }, - .ops = switch (cc) { - else => .m, - .z_and_np => .pseudo_set_z_and_np_m, - .nz_or_p => .pseudo_set_nz_or_p_m, - }, - .data = switch (cc) { - else => .{ .x = .{ - .fixes = .fromCond(cc), - .payload = payload, - } }, - .z_and_np, .nz_or_p => .{ .rx = .{ - .r1 = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to8(), - .payload = payload, - } }, - }, - }); -} - -fn asmJmpReloc(self: *CodeGen, target: Mir.Inst.Index) !Mir.Inst.Index { - return self.addInst(.{ - .tag = .j, - .ops = .inst, - .data = .{ .inst = .{ - .fixes = ._mp, - .inst = target, - } }, - }); -} - -fn asmJccReloc(self: *CodeGen, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index { - return self.addInst(.{ - .tag = switch (cc) { - else => .j, - .z_and_np, .nz_or_p => .pseudo, - }, - .ops = switch (cc) { - else => .inst, - .z_and_np => .pseudo_j_z_and_np_inst, - .nz_or_p => .pseudo_j_nz_or_p_inst, - }, - .data = .{ .inst = .{ - .fixes = switch (cc) { - else => .fromCond(cc), - .z_and_np, .nz_or_p => ._, - }, - .inst = target, - } }, - }); -} - -fn asmReloc(self: *CodeGen, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .inst, - .data = .{ .inst = .{ - .fixes = tag[0], - .inst = target, - } }, - }); -} - -fn asmPlaceholder(self: *CodeGen) !Mir.Inst.Index { - return self.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dead_none, - .data = undefined, - }); -} - -fn asmOpOnly(self: *CodeGen, tag: Mir.Inst.FixedTag) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .none, - .data = .{ .none = .{ - .fixes = tag[0], - } }, - }); -} - -fn asmPseudo(self: *CodeGen, ops: Mir.Inst.Ops) !void { - assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and - std.mem.endsWith(u8, @tagName(ops), "_none")); - _ = try self.addInst(.{ - .tag = .pseudo, - .ops = ops, - .data = undefined, - }); -} - -fn asmPseudoRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register) !void { - assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and - std.mem.endsWith(u8, @tagName(ops), "_r")); - _ = try self.addInst(.{ - .tag = .pseudo, - .ops = ops, - .data = .{ .r = .{ .r1 = reg } }, - }); -} - -fn asmPseudoImmediate(self: *CodeGen, ops: Mir.Inst.Ops, imm: Immediate) !void { - assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and - std.mem.endsWith(u8, @tagName(ops), "_i_s")); - _ = try self.addInst(.{ - .tag = .pseudo, - .ops = ops, - .data = .{ .i = .{ .i = @bitCast(imm.signed) } }, - }); -} - -fn asmPseudoRegisterRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void { - assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and - std.mem.endsWith(u8, @tagName(ops), "_rr")); - _ = try self.addInst(.{ - .tag = .pseudo, - .ops = ops, - .data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } }, - }); -} - -fn asmPseudoRegisterImmediate(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void { - assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and - std.mem.endsWith(u8, @tagName(ops), "_ri_s")); - _ = try self.addInst(.{ - .tag = .pseudo, - .ops = ops, - .data = .{ .ri = .{ .r1 = reg, .i = @bitCast(imm.signed) } }, - }); -} - -fn asmRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .r, - .data = .{ .r = .{ - .fixes = tag[0], - .r1 = reg, - } }, - }); -} - -fn asmImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = switch (imm) { - .signed => .i_s, - .unsigned => .i_u, - .nav => .nav, - .uav => .uav, - .lazy_sym => .lazy_sym, - .extern_func => .extern_func, - }, - .data = switch (imm) { - .signed, .unsigned => .{ .i = .{ - .fixes = tag[0], - .i = switch (imm) { - .signed => |s| @bitCast(s), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - } }, - .nav => |nav| switch (tag[0]) { - ._ => .{ .nav = nav }, - else => unreachable, - }, - .uav => |uav| switch (tag[0]) { - ._ => .{ .uav = uav }, - else => unreachable, - }, - .lazy_sym => |lazy_sym| switch (tag[0]) { - ._ => .{ .lazy_sym = lazy_sym }, - else => unreachable, - }, - .extern_func => |extern_func| switch (tag[0]) { - ._ => .{ .extern_func = extern_func }, - else => unreachable, - }, - }, - }); -} - -fn asmImmediateRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate, reg: Register) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .ir, - .data = .{ .ri = .{ - .fixes = tag[0], - .r1 = reg, - .i = @as(u8, switch (imm) { - .signed => |s| @bitCast(@as(i8, @intCast(s))), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }), - } }, - }); -} - -fn asmImmediateImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm1: Immediate, imm2: Immediate) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .ii, - .data = .{ .ii = .{ - .fixes = tag[0], - .i1 = switch (imm1) { - .signed => |s| @bitCast(@as(i16, @intCast(s))), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - .i2 = switch (imm2) { - .signed => |s| @bitCast(@as(i8, @intCast(s))), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - } }, - }); -} - -fn asmRegisterRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rr, - .data = .{ .rr = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - } }, - }); -} - -fn asmRegisterImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void { - const ops: Mir.Inst.Ops, const i: u32 = switch (imm) { - .signed => |s| .{ .ri_s, @bitCast(s) }, - .unsigned => |u| if (std.math.cast(u32, u)) |small| - .{ .ri_u, small } - else - .{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) }, - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }; - _ = try self.addInst(.{ - .tag = tag[1], - .ops = ops, - .data = .{ .ri = .{ - .fixes = tag[0], - .r1 = reg, - .i = i, - } }, - }); -} - -fn asmRegisterRegisterRegister( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - reg3: Register, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rrr, - .data = .{ .rrr = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .r3 = reg3, - } }, - }); -} - -fn asmRegisterRegisterRegisterRegister( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - reg3: Register, - reg4: Register, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rrrr, - .data = .{ .rrrr = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .r3 = reg3, - .r4 = reg4, - } }, - }); -} - -fn asmRegisterRegisterRegisterImmediate( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - reg3: Register, - imm: Immediate, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rrri, - .data = .{ .rrri = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .r3 = reg3, - .i = switch (imm) { - .signed => |s| @bitCast(@as(i8, @intCast(s))), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - } }, - }); -} - -fn asmRegisterRegisterImmediate( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - imm: Immediate, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = switch (imm) { - .signed => .rri_s, - .unsigned => .rri_u, - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - .data = .{ .rri = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .i = switch (imm) { - .signed => |s| @bitCast(s), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - } }, - }); -} - -fn asmRegisterRegisterMemory( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - m: Memory, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rrm, - .data = .{ .rrx = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmRegisterRegisterMemoryRegister( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - m: Memory, - reg3: Register, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rrmr, - .data = .{ .rrrx = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .r3 = reg3, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .m, - .data = .{ .x = .{ - .fixes = tag[0], - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmRegisterMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rm, - .data = .{ .rx = .{ - .fixes = tag[0], - .r1 = reg, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmRegisterMemoryRegister( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - m: Memory, - reg2: Register, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rmr, - .data = .{ .rrx = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmRegisterMemoryImmediate( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg: Register, - m: Memory, - imm: Immediate, -) !void { - if (switch (imm) { - .signed => |s| if (std.math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null, - .unsigned => |u| std.math.cast(u16, u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }) |small_imm| { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rmi, - .data = .{ .rix = .{ - .fixes = tag[0], - .r1 = reg, - .i = small_imm, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); - } else { - const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { - .signed => |s| @bitCast(s), - .unsigned => |u| @as(u32, @intCast(u)), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - } }); - assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); - _ = try self.addInst(.{ - .tag = tag[1], - .ops = switch (imm) { - .signed => .rmi_s, - .unsigned => .rmi_u, - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - .data = .{ .rx = .{ - .fixes = tag[0], - .r1 = reg, - .payload = payload, - } }, - }); - } -} - -fn asmRegisterRegisterMemoryImmediate( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - reg1: Register, - reg2: Register, - m: Memory, - imm: Immediate, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .rrmi, - .data = .{ .rrix = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .i = @intCast(imm.unsigned), - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmMemoryRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .mr, - .data = .{ .rx = .{ - .fixes = tag[0], - .r1 = reg, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmMemoryImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void { - const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { - .signed => |s| @bitCast(s), - .unsigned => |u| @intCast(u), - .nav, .uav, .lazy_sym, .extern_func => unreachable, - } }); - assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); - _ = try self.addInst(.{ - .tag = tag[1], - .ops = switch (imm) { - .signed => .mi_s, - .unsigned => .mi_u, - .nav, .uav, .lazy_sym, .extern_func => unreachable, - }, - .data = .{ .x = .{ - .fixes = tag[0], - .payload = payload, - } }, - }); -} - -fn asmMemoryRegisterRegister( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - m: Memory, - reg1: Register, - reg2: Register, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .mrr, - .data = .{ .rrx = .{ - .fixes = tag[0], - .r1 = reg1, - .r2 = reg2, - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn asmMemoryRegisterImmediate( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - m: Memory, - reg: Register, - imm: Immediate, -) !void { - _ = try self.addInst(.{ - .tag = tag[1], - .ops = .mri, - .data = .{ .rix = .{ - .fixes = tag[0], - .r1 = reg, - .i = @intCast(imm.unsigned), - .payload = try self.addExtra(Mir.Memory.encode(m)), - } }, - }); -} - -fn gen( - self: *CodeGen, - zir: *const std.zig.Zir, - func_zir_inst: std.zig.Zir.Inst.Index, - comptime_args: InternPool.Index.Slice, - air_arg_count: u32, -) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const fn_info = zcu.typeToFunc(self.fn_type).?; - if (fn_info.cc != .naked) { - try self.asmRegister(.{ ._, .push }, .rbp); - try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8)); - try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0)); - try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp); - try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp); - const backpatch_push_callee_preserved_regs = try self.asmPlaceholder(); - const backpatch_frame_align = try self.asmPlaceholder(); - const backpatch_frame_align_extra = try self.asmPlaceholder(); - const backpatch_stack_alloc = try self.asmPlaceholder(); - const backpatch_stack_alloc_extra = try self.asmPlaceholder(); - - switch (self.ret_mcv.long) { - .none, .unreach => {}, - .indirect => { - // The address where to store the return value for the caller is in a - // register which the callee is free to clobber. Therefore, we purposely - // spill it to stack immediately. - const frame_index = try self.allocFrameIndex(.initSpill(.usize, zcu)); - try self.genSetMem( - .{ .frame = frame_index }, - 0, - .usize, - self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off), - .{}, - ); - self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } }; - tracking_log.debug("spill {f} to {f}", .{ self.ret_mcv.long, frame_index }); - }, - else => unreachable, - } - - if (fn_info.is_var_args) switch (fn_info.cc) { - .x86_64_sysv => { - const info = &self.va_info.sysv; - const reg_save_area_fi = try self.allocFrameIndex(.init(.{ - .size = abi.SysV.c_abi_int_param_regs.len * 8 + - abi.SysV.c_abi_sse_param_regs.len * 16, - .alignment = .@"16", - })); - info.reg_save_area = .{ .index = reg_save_area_fi }; - - for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) |reg, reg_i| - try self.genSetMem(.{ .frame = reg_save_area_fi }, @intCast(reg_i * 8), .usize, .{ .register = reg }, .{}); - - try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count)); - const skip_sse_reloc = try self.asmJccReloc(.na, undefined); - - const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type }); - for (abi.SysV.c_abi_sse_param_regs[info.fp_count..], info.fp_count..) |reg, reg_i| - try self.genSetMem( - .{ .frame = reg_save_area_fi }, - @intCast(abi.SysV.c_abi_int_param_regs.len * 8 + reg_i * 16), - vec_2_f64, - .{ .register = reg }, - .{}, - ); - - self.performReloc(skip_sse_reloc); - }, - .x86_64_win => return self.fail("TODO implement gen var arg function for Win64", .{}), - else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}), - }; - - if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_prologue_end_none); - - try self.genMainBody(zir, func_zir_inst, comptime_args, air_arg_count); - - const epilogue = if (self.epilogue_relocs.items.len > 0) epilogue: { - var last_inst: Mir.Inst.Index = @intCast(self.mir_instructions.len - 1); - while (self.epilogue_relocs.getLastOrNull() == last_inst) { - self.epilogue_relocs.items.len -= 1; - self.mir_instructions.set(last_inst, .{ - .tag = .pseudo, - .ops = .pseudo_dead_none, - .data = undefined, - }); - last_inst -= 1; - } - for (self.epilogue_relocs.items) |epilogue_reloc| self.performReloc(epilogue_reloc); - - if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_epilogue_begin_none); - const backpatch_stack_dealloc = try self.asmPlaceholder(); - const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder(); - try self.asmRegister(.{ ._, .pop }, .rbp); - try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8)); - try self.asmOpOnly(.{ ._, .ret }); - break :epilogue .{ - .backpatch_stack_dealloc = backpatch_stack_dealloc, - .backpatch_pop_callee_preserved_regs = backpatch_pop_callee_preserved_regs, - }; - } else null; - - const frame_layout = try self.computeFrameLayout(fn_info.cc); - const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32); - const need_stack_adjust = frame_layout.stack_adjust > 0; - const need_save_reg = frame_layout.save_reg_list.count() > 0; - if (need_frame_align) { - const page_align = @as(u32, std.math.maxInt(u32)) << 12; - self.mir_instructions.set(backpatch_frame_align, .{ - .tag = .@"and", - .ops = .ri_s, - .data = .{ .ri = .{ - .r1 = .rsp, - .i = @max(frame_layout.stack_mask, page_align), - } }, - }); - if (frame_layout.stack_mask < page_align) { - self.mir_instructions.set(backpatch_frame_align_extra, .{ - .tag = .pseudo, - .ops = .pseudo_probe_align_ri_s, - .data = .{ .ri = .{ - .r1 = .rsp, - .i = ~frame_layout.stack_mask & page_align, - } }, - }); - } - } - if (need_stack_adjust) { - const page_size: u32 = 1 << 12; - if (frame_layout.stack_adjust <= page_size) { - self.mir_instructions.set(backpatch_stack_alloc, .{ - .tag = .sub, - .ops = .ri_s, - .data = .{ .ri = .{ - .r1 = .rsp, - .i = frame_layout.stack_adjust, - } }, - }); - } else if (frame_layout.stack_adjust < - page_size * Lower.pseudo_probe_adjust_unrolled_max_insts) - { - self.mir_instructions.set(backpatch_stack_alloc, .{ - .tag = .pseudo, - .ops = .pseudo_probe_adjust_unrolled_ri_s, - .data = .{ .ri = .{ - .r1 = .rsp, - .i = frame_layout.stack_adjust, - } }, - }); - } else { - const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc); - self.mir_instructions.set(backpatch_stack_alloc, .{ - .tag = .pseudo, - .ops = .pseudo_probe_adjust_setup_rri_s, - .data = .{ .rri = .{ - .r1 = .rsp, - .r2 = scratch_reg, - .i = frame_layout.stack_adjust, - } }, - }); - self.mir_instructions.set(backpatch_stack_alloc_extra, .{ - .tag = .pseudo, - .ops = .pseudo_probe_adjust_loop_rr, - .data = .{ .rr = .{ - .r1 = .rsp, - .r2 = scratch_reg, - } }, - }); - } - } - if (epilogue) |e| if (need_frame_align or need_stack_adjust) { - self.mir_instructions.set(e.backpatch_stack_dealloc, switch (-frame_layout.save_reg_list.size(self.target)) { - 0 => .{ - .tag = .mov, - .ops = .rr, - .data = .{ .rr = .{ - .r1 = .rsp, - .r2 = .rbp, - } }, - }, - else => |disp| .{ - .tag = .lea, - .ops = .rm, - .data = .{ .rx = .{ - .r1 = .rsp, - .payload = try self.addExtra(Mir.Memory.encode(.{ - .base = .{ .reg = .rbp }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = disp, - } }, - })), - } }, - }, - }); - }; - if (need_save_reg) { - self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{ - .tag = .pseudo, - .ops = .pseudo_push_reg_list, - .data = .{ .reg_list = frame_layout.save_reg_list }, - }); - if (epilogue) |e| self.mir_instructions.set(e.backpatch_pop_callee_preserved_regs, .{ - .tag = .pseudo, - .ops = .pseudo_pop_reg_list, - .data = .{ .reg_list = frame_layout.save_reg_list }, - }); - } - } else { - if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_prologue_end_none); - try self.genMainBody(zir, func_zir_inst, comptime_args, air_arg_count); - if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_epilogue_begin_none); - } -} - -fn genMainBody( - cg: *CodeGen, - zir: *const std.zig.Zir, - func_zir_inst: std.zig.Zir.Inst.Index, - comptime_args: InternPool.Index.Slice, - air_arg_count: u32, -) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const main_body = cg.air.getMainBody(); - const air_args_body = main_body[0..air_arg_count]; - try cg.genBody(air_args_body); - - if (!cg.mod.strip) { - var air_arg_index: usize = 0; - const fn_info = zcu.typeToFunc(cg.fn_type).?; - var fn_param_index: usize = 0; - var zir_param_index: usize = 0; - for (zir.getParamBody(func_zir_inst)) |zir_param_inst| { - const name = switch (zir.getParamName(zir_param_inst) orelse break) { - .empty => .none, - else => |zir_name| try cg.addString(zir.nullTerminatedString(zir_name)), - }; - defer zir_param_index += 1; - - if (comptime_args.len > 0) switch (comptime_args.get(ip)[zir_param_index]) { - .none => {}, - else => |comptime_arg| { - try cg.mir_locals.append(cg.gpa, .{ .name = name, .type = ip.typeOf(comptime_arg) }); - _ = try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_arg_val, - .data = .{ .ip_index = comptime_arg }, - }); - continue; - }, - }; - - const arg_ty = fn_info.param_types.get(ip)[fn_param_index]; - try cg.mir_locals.append(cg.gpa, .{ .name = name, .type = arg_ty }); - fn_param_index += 1; - - if (air_arg_index == air_args_body.len) { - try cg.asmPseudo(.pseudo_dbg_arg_none); - continue; - } - const air_arg_inst = air_args_body[air_arg_index]; - const air_arg_data = cg.air.instructions.items(.data)[air_arg_index].arg; - if (air_arg_data.zir_param_index != zir_param_index) { - try cg.asmPseudo(.pseudo_dbg_arg_none); - continue; - } - air_arg_index += 1; - try cg.genLocalDebugInfo( - .arg, - .fromInterned(arg_ty), - cg.getResolvedInstValue(air_arg_inst).short, - ); - } - if (fn_info.is_var_args) try cg.asmPseudo(.pseudo_dbg_var_args_none); - } - - try cg.genBody(main_body[air_arg_count..]); -} - -fn checkInvariantsAfterAirInst(cg: *CodeGen) void { - assert(!cg.register_manager.lockedRegsExist()); - - if (std.debug.runtime_safety) { - // check consistency of tracked registers - var it = cg.register_manager.free_registers.iterator(.{ .kind = .unset }); - while (it.next()) |index| { - const tracked_inst = cg.register_manager.registers[index]; - const tracking = cg.getResolvedInstValue(tracked_inst); - for (tracking.getRegs()) |reg| { - if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; - } else unreachable; // tracked register not in use - } - } -} - -fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { - if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none); - try cg.genBody(body); - if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_leave_block_none); -} - -fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { - @setEvalBranchQuota(29_500); - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const air_tags = cg.air.instructions.items(.tag); - const air_datas = cg.air.instructions.items(.data); - - for (body) |inst| { - if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue; - wip_mir_log.debug("{f}", .{cg.fmtAir(inst)}); - verbose_tracking_log.debug("{f}", .{cg.fmtTracking()}); - - cg.reused_operands = .initEmpty(); - try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); - switch (air_tags[@intFromEnum(inst)]) { - .select => try cg.airSelect(inst), - .shuffle_one, .shuffle_two => @panic("x86_64 TODO: shuffle_one/shuffle_two"), - - .arg => try cg.airArg(inst), - .add, .add_optimized, .add_wrap => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .imm8, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .remainder_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp1d, .memsia(.src1d, .@"4", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .add, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .add, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .add, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .add, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, - .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .add, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .add, .dst0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .add, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .add, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .add, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .add, .dst0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_, .add, .src1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .add, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .add, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .add, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .add, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .add, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - switch (air_tag) { - else => unreachable, - .add, .add_optimized => {}, - .add_wrap => res[0].wrapInt(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - res[0].tracking(cg), - }), - else => |e| return e, - }, - } - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .add_safe => unreachable, - .add_sat => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"or", .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._po, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._pe, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .add, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .cmov, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._rx, .ro, .tmp1q, .dst0q, .sia(-31, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .sia(-31, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .sia(-31, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._a, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ .@"0:", ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp2q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp2q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp2q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .@"or", .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .@"or", .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .@"or", .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .@"or", .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .@"or", .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .adds, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .adds, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .adds, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .adds, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .adds, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .adds, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_32_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .addus, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .addus, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .addus, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .addus, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_b, .minu, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .adds, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .adds, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .adds, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .adds, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, - .{ .type = .vector_4_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, .p_w, .maxs, .dst0q, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_w, .mins, .dst0q, .lea(.tmp0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_w, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_w, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_16_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .addus, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .addus, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .addus, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .addus, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_w, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .umax_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .addus, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_d, .sra, .tmp2x, .src0x, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3x, .src0x, .dst0x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, - .{ ._, .p_d, .sra, .tmp2x, .ui(31), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_d, .sra, .tmp2y, .src0y, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3y, .src0y, .dst0y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_d, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_d, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_d, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .cmpeq, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .src1x, ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_d, .minu, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_d, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .cmpeq, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vp_, .xor, .dst0y, .src0y, .dst0y, ._ }, - .{ ._, .vp_d, .minu, .dst0y, .dst0y, .src1y, ._ }, - .{ ._, .vp_d, .add, .dst0y, .dst0y, .src0y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_d, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_d, .minu, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .src0x, .dst0x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .src0y, .dst0y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp2x, .dst0x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp2x, .dst0x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - .{ ._, .p_q, .cmpeq, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - .{ ._, .vp_q, .cmpeq, .tmp3y, .tmp3y, .tmp3y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .tmp2y, .dst0y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .src0x, .tmp2x, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .src0x, .tmp2x, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .src0y, .tmp2y, ._ }, - .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .tmp3y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .lea(.tmp0x), .tmp2x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .lea(.tmp0x), .tmp2x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .lea(.tmp0x), .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp2y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .lea(.tmp0y), .tmp2y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .@"or", .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .@"or", .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .@"or", .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .@"or", .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .adds, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .adds, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .adds, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i8, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .adds, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_b, .broadcast, .tmp3y, .lea(.tmp0b), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_b, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_b, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .maxs, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .mins, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .adds, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .adds, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .addus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .addus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .addus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .addus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"or", .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"or", .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .broadcast, .tmp2y, .lea(.tmp0b), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .minu, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .minu, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .minu, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, - .{ .type = .vector_8_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._q, .mov, .tmp2q, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .minu, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2b, .tmp1b, ._, ._ }, - .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2b, .tmp1b, ._, ._ }, - .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .adds, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .adds, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .adds, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_w, .broadcast, .tmp3y, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_w, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_w, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_w, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_w, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .maxs, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .mins, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .addus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .addus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .addus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .addus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u16, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .addus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"or", .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_w, .minu, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_w, .minu, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .minu, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .umax_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .addus, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .sub, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_d, .sra, .tmp6y, .tmp3y, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3y, .tmp3y, .tmp5y, ._ }, - .{ ._, .vp_, .xor, .tmp6y, .tmp6y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_ps, .blendv, .tmp3y, .tmp5y, .tmp6y, .tmp3y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp5x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_d, .sra, .tmp6x, .tmp3x, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3x, .tmp3x, .tmp5x, ._ }, - .{ ._, .vp_, .xor, .tmp6x, .tmp6x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .v_ps, .blendv, .tmp3x, .tmp5x, .tmp6x, .tmp3x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .add, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .sra, .tmp6x, .ui(31), ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp3x, .tmp5x, ._, ._ }, - .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .blendv, .tmp5x, .tmp6x, .tmp3x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp5x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smin), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_d, .broadcast, .tmp3y, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_d, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_d, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .maxs, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .mins, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp2y, .tmp1y, ._ }, - .{ ._, .vp_d, .minu, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .add, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp2x, .tmp1x, ._ }, - .{ ._, .vp_d, .minu, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .add, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .add, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .dword, .is = 32 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .dword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"or", .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .minu, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .minu, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .minu, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp5y, ._ }, - .{ ._, .vp_q, .add, .tmp6y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp3y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp3y, .tmp6y, ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_pd, .blendv, .tmp5y, .tmp6y, .tmp5y, .tmp3y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp5y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp5x, ._ }, - .{ ._, .vp_q, .add, .tmp6x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp3x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp6x, ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .v_pd, .blendv, .tmp5x, .tmp6x, .tmp5x, .tmp3x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp5x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp5x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .add, .tmp6x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp6x, ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._pd, .blendv, .tmp6x, .tmp5x, .tmp3x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp6x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp3y, .tmp5y }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp2y, .tmp5y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp3x, .tmp5x }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp2x, .tmp5x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_q, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp4x, .tmp3x, .tmp5x, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp4x, .tmp2x, .tmp5x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .cmov, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._rx, .ro, .tmp4q, .tmp2q, .sia(-31, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp4y, .tmp5y, ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp4x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp4x, .tmp5x, ._ }, - .{ ._, .vp_, .@"or", .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp5x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp4y, ._ }, - .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp2y, .tmp4y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp4x, .tmp4x, .tmp4x, ._ }, - .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp2x, .tmp4x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp3x, .tmp2x, .tmp4x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp2y, .tmp4y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp2x, .tmp4x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp3x, .tmp2x, .tmp4x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._a, .cmov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp5p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .adc, .tmp6q, .leasi(.tmp3q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp4q, .@"8", .tmp5), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sia(2, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp3q, 8), .tmp5q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp5p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .adc, .tmp6q, .leasi(.tmp3q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp4q, .@"8", .tmp5), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.tmp4q), .si(0), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, - .{ .@"1:", ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .si(-1), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp6p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp7q, .leasi(.tmp3q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .leasi(.tmp4q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp5q, .@"8", .tmp6), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp6p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp5q, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .leaad(.tmp5, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.tmp5q), .si(0), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp3, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .si(-1), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp7q, .leasi(.tmp3q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .leasi(.tmp4q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp5q, .@"8", .tmp6), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp6p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .leaa(.tmp5, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp5q, -8), .tmp1q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .sub, .sub_optimized, .sub_wrap => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .remainder_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1d, .memsia(.src1d, .@"4", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .sub, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .sub, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .sub, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .sub, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .sub, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, - .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ }, - .{ ._, ._, .sbb, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .sub, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .sub, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .sub, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .sub, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .sub, .dst0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .sub, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .sub, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .sub, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .sub, .dst0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_, .sub, .src1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .sub, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .sub, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .sub, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .sub, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, .f_p, .sub, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .sub, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .sub, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - switch (air_tag) { - else => unreachable, - .sub, .sub_optimized => {}, - .sub_wrap => res[0].wrapInt(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - res[0].tracking(cg), - }), - else => |e| return e, - }, - } - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .sub_safe => unreachable, - .sub_sat => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 0 }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .imm = 0 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .imm = 0 }, .to_mut_gpr, .none } }, - .{ .src = .{ .any, .imm8, .none } }, - }, - .dst_temps = .{ .{ .imm = 0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .bmi, null, null, null }, - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .andn, .dst0d, .src1d, .src0d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_mut_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .src1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._po, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._pe, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .add, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .cmov, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._rx, .ro, .tmp1q, .dst0q, .sia(-31, .src0, .add_bit_size), ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .sia(-31, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .sia(-31, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._c, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ .@"0:", ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp2q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_mut_mmx, .none } }, - .{ .src = .{ .to_mmx, .to_mut_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .andn, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .andn, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_mut_sse, .none } }, - .{ .src = .{ .to_sse, .to_mut_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_mut_sse, .none } }, - .{ .src = .{ .to_sse, .to_mut_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .andn, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .andn, .dst0y, .src1y, .src0y, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .subs, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .subs, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .subs, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .subs, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .subs, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .adds, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_32_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .subus, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .subus, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .subus, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .subus, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .subs, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .subs, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .subs, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .subs, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, - .{ .type = .vector_4_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, .p_w, .maxs, .dst0q, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_w, .mins, .dst0q, .lea(.tmp0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_w, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_w, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_16_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .subus, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .subus, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .subus, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .subus, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_d, .sra, .tmp2x, .src0x, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3x, .dst0x, .src0x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, - .{ ._, .p_d, .sra, .tmp2x, .ui(31), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp3x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_d, .sra, .tmp2y, .src0y, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3y, .dst0y, .src0y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_d, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_d, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_d, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .maxu, .dst0x, .src1x, .src0x, ._ }, - .{ ._, .vp_d, .sub, .dst0x, .dst0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .maxu, .src0x, .src1x, ._, ._ }, - .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .maxu, .dst0y, .src1y, .src0y, ._ }, - .{ ._, .vp_d, .sub, .dst0y, .dst0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .src0x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .dst0y, .src0y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - .{ ._, .p_q, .cmpeq, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - .{ ._, .vp_q, .cmpeq, .tmp3y, .tmp3y, .tmp3y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .src1x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .src1x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .src0y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .src1y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .tmp2y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp0x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .tmp0y, .tmp0y, .tmp0y, ._ }, - .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp0y, .dst0y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .andn, .tmp1y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .andn, .tmp1x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .andn, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .subs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .subs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .subs, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i8, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .subs, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_b, .broadcast, .tmp3y, .lea(.tmp0b), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_b, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_b, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .maxs, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .mins, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .subs, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .adds, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .subus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_b, .subus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .subus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .subs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .subs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .subs, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_w, .broadcast, .tmp3y, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_w, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_w, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_w, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_w, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .maxs, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .mins, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .subus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .subus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .subus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u16, .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .subus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sub, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_d, .sra, .tmp6y, .tmp3y, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3y, .tmp5y, .tmp3y, ._ }, - .{ ._, .vp_, .xor, .tmp6y, .tmp6y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_ps, .blendv, .tmp3y, .tmp5y, .tmp6y, .tmp3y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sub, .tmp5x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_d, .sra, .tmp6x, .tmp3x, .ui(31), ._ }, - .{ ._, .vp_d, .cmpgt, .tmp3x, .tmp5x, .tmp3x, ._ }, - .{ ._, .vp_, .xor, .tmp6x, .tmp6x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .v_ps, .blendv, .tmp3x, .tmp5x, .tmp6x, .tmp3x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .sub, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .sra, .tmp6x, .ui(31), ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp3x, .tmp5x, ._, ._ }, - .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .blendv, .tmp6x, .tmp5x, .tmp3x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp6x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smin), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_d, .broadcast, .tmp3y, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, - .{ ._, .vp_d, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, - .{ ._, .vp_d, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .maxs, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .mins, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .maxu, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .sub, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .maxu, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_d, .sub, .tmp1x, .tmp2x, .tmp1x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .maxu, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .sub, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp5y, ._ }, - .{ ._, .vp_q, .sub, .tmp6y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp3y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp6y, .tmp3y, ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_pd, .blendv, .tmp5y, .tmp6y, .tmp5y, .tmp3y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp5y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp5x, ._ }, - .{ ._, .vp_q, .sub, .tmp6x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp3x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp6x, .tmp3x, ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .v_pd, .blendv, .tmp5x, .tmp6x, .tmp5x, .tmp3x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp5x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp5x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .sub, .tmp6x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp7x, .tmp6x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp7x, .tmp3x, ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp7x, .tmp4x, ._, ._ }, - .{ ._, ._pd, .blendv, .tmp6x, .tmp5x, .tmp7x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp6x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, - .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp3y, .tmp5y }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp2y, .tmp5y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp3x, .tmp5x }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp2x, .tmp5x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_q, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp4x, .tmp3x, .tmp5x, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp4x, .tmp2x, .tmp5x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .cmov, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._rx, .ro, .tmp4q, .tmp2q, .sia(-31, .src0, .add_bit_size_rem_64), ._ }, - .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp5y, .tmp3y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp5x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp4x, .tmp4x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_, .@"and", .tmp3x, .tmp5x, .tmp3x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .sub, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp5x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._c, .cmov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .blendv, .tmp2y, .tmp2y, .tmp1y, .tmp2y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_q, .sub, .tmp2x, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .blendv, .tmp2x, .tmp2x, .tmp1x, .tmp2x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .sub, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .blendv, .tmp2x, .tmp1x, .tmp2x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp1q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._c, .cmov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sia(2, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp3q, 8), .tmp5q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, - .{ .@"1:", ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, - .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp3, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp5p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sbb, .tmp6q, .leasi(.tmp3q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp4q, .@"8", .tmp5), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .mul, .mul_optimized => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const ty = cg.typeOf(bin_op.lhs); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ }, - .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ }, - .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp0x, .src1q, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .p_w, .movsxb, .tmp0x, .src1q, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp0x, .src1x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp0y, .src1x, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp0x, .src1q, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .p_w, .movzxb, .tmp0x, .src1q, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp0y, .src1x, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .tmp1y, .ui(1), ._ }, - .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp3x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .tmp1y, .ui(1), ._ }, - .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .ackusw, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .ackusw, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .mull, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .mull, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .mull, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .mull, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .mull, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ }, - .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ }, - .{ ._, ._z, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp9q, .tmp8q, .leai(.tmp2q, .tmp6), ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .leaiad(.tmp3q, .tmp6, .add_dst0_elem_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp9q, .leaia(.tmp3q, .tmp6, .add_dst0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .lead(.tmp6, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"2b", ._, ._, ._ }, - .{ .@"3:", ._, .mov, .leai(.tmp3q, .tmp4), .tmp5q, ._, ._ }, - .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_, .mul, .src1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .mul_safe => unreachable, - .mul_wrap => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const ty = cg.typeOf(bin_op.lhs); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ }, - .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ }, - .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ }, - .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp2x, .src1q, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .p_w, .movsxb, .tmp2x, .src1q, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .src1x, .src1x, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp1x), ._, ._ }, - .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp2y, .src1x, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ }, - .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp2x, .src1q, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ }, - .{ ._, .p_w, .movzxb, .tmp2x, .src1q, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp2y, .src1x, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ }, - .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movsxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .mull, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .mull, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .mull, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .mull, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .mull, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .mul, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ }, - .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ }, - .{ ._, ._z, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp9q, .tmp8q, .leai(.tmp2q, .tmp6), ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .leaiad(.tmp3q, .tmp6, .add_dst0_elem_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp9q, .leaia(.tmp3q, .tmp6, .add_dst0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .lead(.tmp6, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"2b", ._, ._, ._ }, - .{ .@"3:", ._, .mov, .leai(.tmp3q, .tmp4), .tmp5q, ._, ._ }, - .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_, .mul, .src1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - res[0].wrapInt(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - res[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .mul_sat => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .gphi } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .dst0h, ._, ._ }, - .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ }, - .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ }, - .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi, .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ }, - .{ ._, ._, .@"or", .tmp2w, .tmp0w, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi, .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ }, - .{ ._, ._, .@"or", .tmp2d, .tmp0d, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .cmov, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .andn, .tmp2q, .tmp1q, .dst0q, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, - .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .div_float, .div_float_optimized, .div_exact, .div_exact_optimized => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const ty = cg.typeOf(bin_op.lhs); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - (if (cg.floatBits(ty.scalarType(zcu))) |_| cg.select(&res, &.{ty}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_, .div, .src1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .divr, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .divr, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .div, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .div, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) else err: { - assert(air_tag == .div_exact); - res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err; - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .div_trunc => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const ty = cg.typeOf(bin_op.lhs); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - (if (cg.floatBits(ty.scalarType(zcu))) |_| cg.select(&res, &.{ty}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .zero, .precision = .inexact }) }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .tmp1x, .tmp1y, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .zero, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - .{ ._, ._ss, .round, .dst0x, .dst0d, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "truncf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "truncf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ss, .div, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, - .{ ._, .v_sd, .round, .dst0x, .dst0x, .dst0q, .rm(.{ .direction = .zero, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - .{ ._, ._sd, .round, .dst0x, .dst0q, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_pd, .round, .dst0y, .dst0y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sd, .div, .tmp1x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3p, .tmp2p, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .mem(.dst0), ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp1, .add_unaligned_size), ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp4t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5p, .tmp4p, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) else err: { - res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err; - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .div_trunc_optimized, .div_floor_optimized => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(bits.RoundMode.Direction, switch (air_tag) { - else => unreachable, - .div_trunc_optimized => .zero, - .div_floor_optimized => .down, - })) { - else => unreachable, - inline .zero, .down => |direction| comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }) }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__trunch", - .down => "__floorh", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__trunch", - .down => "__floorh", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__trunch", - .down => "__floorh", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__trunch", - .down => "__floorh", - } } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__trunch", - .down => "__floorh", - } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - .{ ._, ._ss, .round, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "truncf", - .down => "floorf", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "truncf", - .down => "floorf", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ss, .div, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .round, .tmp1x, .tmp1x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, - .{ ._, .v_sd, .round, .dst0x, .dst0x, .dst0q, .rm(.{ .direction = direction, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - .{ ._, ._sd, .round, .dst0x, .dst0q, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "trunc", - .down => "floor", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "trunc", - .down => "floor", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_pd, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_pd, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .round, .tmp1x, .tmp1x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "trunc", - .down => "floor", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sd, .div, .tmp1x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "trunc", - .down => "floor", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__truncx", - .down => "__floorx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__truncx", - .down => "__floorx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3p, .tmp2p, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__truncx", - .down => "__floorx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .mem(.dst0), ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__truncx", - .down => "__floorx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__truncx", - .down => "__floorx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "__truncx", - .down => "__floorx", - } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp1, .add_unaligned_size), ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp4t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5p, .tmp4p, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "truncq", - .down => "floorq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "truncq", - .down => "floorq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "truncq", - .down => "floorq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .zero => "truncq", - .down => "floorq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .div_floor => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const ty = cg.typeOf(bin_op.lhs); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - (if (cg.intInfo(ty)) |int_info| err: switch (int_info.signedness) { - .signed => cg.select(&res, &.{ty}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 1 }, .{ .exact_signed_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .any, .any, .none } }, - }, - .dst_temps = .{ .{ .imm = 0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_mut_gphi, .none } }, - .{ .src = .{ .to_gpr, .to_mut_gphi, .none } }, - }, - .dst_temps = .{ .{ .reg = .al }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, - .{ ._, .i_, .div, .src1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .src1b, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1b, .dst0h, ._, ._ }, - .{ ._, ._, .cmp, .src1b, .sia(1, .src0, .add_smin), ._, ._ }, - .{ ._, ._, .adc, .dst0b, .si(-1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - .{ ._, ._, .@"and", .src1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1d, .tmp0d, ._, ._ }, - .{ ._, ._, .cmp, .src1w, .sia(1, .src0, .add_smin), ._, ._ }, - .{ ._, ._, .adc, .dst0d, .si(-1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1d, .tmp0d, ._, ._ }, - .{ ._, ._, .cmp, .src1w, .sia(1, .src0, .add_smin), ._, ._ }, - .{ ._, ._, .adc, .dst0d, .si(-1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .to_mut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1d, ._, ._, ._ }, - .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1d, .tmp0d, ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(1, .src0, .add_smin), ._, ._ }, - .{ ._, ._, .adc, .dst0d, .si(-1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .to_mut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .src1q, .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .src1q, .tmp0q, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .src1q, .tmp1q, ._, ._ }, - .{ ._, ._, .adc, .dst0q, .si(-1), ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q0, .mem(.src0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0q1, .memd(.src0q, 8), ._, ._ }, - .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q0, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp0q1, .tmp0q0, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q1, .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp0q1, .dst0q1, ._, ._ }, - .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp0q1, .tmp0q0, ._, ._ }, - .{ ._, ._, .sbb, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q0, .mem(.src0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0q1, .memd(.src0q, 8), ._, ._ }, - .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0q0, .tmp3q, ._, ._ }, - .{ ._, ._, .sbb, .dst0q1, .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divei4" } }, - .{ .kind = .{ .mem_of_type = .dst0 } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .memad(.src0d, .add_size, -4), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .memad(.src1d, .add_size, -4), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"1:", ._, .cmp, .memi(.tmp6q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._e, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_size_div_8), ._, ._ }, - .{ .@"2:", ._, .adc, .memsia(.dst0q, .@"8", .tmp0, .add_size), .si(-1), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"3:", ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - } }, - } }), - .unsigned => res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err, - } else cg.select(&res, &.{ty}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .down, .precision = .inexact }) }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, - .{ ._, .v_, .cvtps2ph, .tmp1x, .tmp1y, .rm(.{}), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .down, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - .{ ._, ._ss, .round, .dst0x, .dst0d, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "floorf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "floorf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ss, .div, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, - .{ ._, .v_sd, .round, .dst0x, .dst0x, .dst0q, .rm(.{ .direction = .down, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - .{ ._, ._sd, .round, .dst0x, .dst0q, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .v_pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .v_pd, .round, .dst0y, .dst0y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_pd, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sd, .div, .tmp1x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3p, .tmp2p, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .mem(.dst0), ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_cc_abi = .win64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .div, ._, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp1, .add_unaligned_size), ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp4t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5p, .tmp4p, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, - .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } })) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .rem, .rem_optimized => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .any, .any, .none } }, - }, - .dst_temps = .{ .{ .imm = 0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .ah }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, - .{ ._, .i_, .div, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .ah }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .div, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .div, .src1w, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .edx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .edx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .div, .src1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .rdx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .rdx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._, .div, .src1q, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__umodei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .reg = .ax } }, - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .eax } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__umodei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, .x87, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, .x87, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, .x87, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, .x87, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .mod, .mod_optimized => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .any, .any, .none } }, - }, - .dst_temps = .{ .{ .imm = 0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_mut_gphi, .none } }, - .{ .src = .{ .to_gpr, .to_mut_gphi, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .gphi } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .ah }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, - .{ ._, .i_, .div, .src1b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .tmp0h, .src1b, ._, ._ }, - .{ ._, ._, .@"and", .src1b, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1b, .dst0h, ._, ._ }, - .{ ._, ._, .cmp, .src1b, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gphi, .none } }, - .{ .src = .{ .to_gpr, .to_gphi, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .gphi } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .ah }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, - .{ ._, .i_, .div, .src1b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .dst0h, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .dst0h, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .ah }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .div, .src1b, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .src0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .src1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .src0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .src0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .src0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .dx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .div, .src1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .edx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1d, ._, ._, ._ }, - .{ ._, ._, .mov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .add, .src0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src1d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .edx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1d, ._, ._, ._ }, - .{ ._, ._, .mov, .src0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src0d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .edx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .div, .src1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .to_mut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .rdx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .mov, .src0q, .dst0q, ._, ._ }, - .{ ._, ._, .add, .src0q, .src1q, ._, ._ }, - .{ ._, ._, .@"and", .src1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .src1q, .dst0q, ._, ._ }, - .{ ._, ._, .cmp, .src1q, .tmp0q, ._, ._ }, - .{ ._, ._a, .cmov, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .rdx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .mov, .src0q, .src1q, ._, ._ }, - .{ ._, ._, .@"and", .src0q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .src0q, .dst0q, ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .rdx }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._, .div, .src1q, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .src0q0, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .mov, .src0q1, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q0, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .src0q1, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .dst0q1, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp0q, .src0q1, ._, ._ }, - .{ ._, ._nae, .cmov, .src0q0, .tmp2q, ._, ._ }, - .{ ._, ._ae, .cmov, .tmp2q, .mem(.src1q), ._, ._ }, - .{ ._, ._, .add, .dst0q0, .tmp2q, ._, ._ }, - .{ ._, ._, .adc, .dst0q1, .src0q0, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .src0q0, .memd(.src1q, 8), ._, ._ }, - .{ ._, ._, .mov, .src0q1, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q0, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .src0q1, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .dst0q1, ._, ._ }, - .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp0q, .src0q1, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .dst0q0, .mem(.src1q), ._, ._ }, - .{ ._, ._, .adc, .dst0q1, .src0q0, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-3, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .cmp, .memad(.dst0q, .add_size, -16), .si(1), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3q, .memsi(.dst0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .sbb, .tmp3q, .si(0), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src1, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._ns, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__umodei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .src0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .src1x, .src1q, ._, ._ }, - .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .src0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .src1x, .src1q, ._, ._ }, - .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp0d), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .mem(.tmp0d), ._, ._ }, - .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._ss, .mov, .src1x, .mem(.tmp0d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp0d), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .mem(.tmp0d), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._ss, .mov, .src1x, .mem(.tmp0d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .v_ss, .add, .tmp1x, .tmp1x, .tmp3d, ._ }, - .{ ._, .v_, .cvtps2ph, .tmp1q, .tmp1x, .rm(.{}), ._ }, - .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .v_ss, .add, .tmp1x, .tmp1x, .tmp3d, ._ }, - .{ ._, .v_, .cvtps2ph, .tmp1q, .tmp1x, .rm(.{}), ._ }, - .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f16, .kind = .{ .reg = .dx } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .mem(.tmp6d), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp1d, .mem(.tmp6d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .mem(.tmp6d), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp1d, .mem(.tmp6d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .{ .type = .f32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .{ .type = .f32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._d, .mov, .tmp0d, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp2d, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .cmp, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._d, .mov, .src1x, .tmp0d, ._, ._ }, - .{ ._, ._ss, .add, .dst0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp0d), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .cmp, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._ss, .add, .dst0x, .mem(.tmp0d), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .{ .type = .f32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_d, .mov, .tmp3d, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp3d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_d, .mov, .tmp2x, .tmp3d, ._, ._ }, - .{ ._, .v_ss, .add, .tmp1x, .tmp1x, .tmp2d, ._ }, - .{ .@"1:", .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .{ .type = .f32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._d, .mov, .tmp3d, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp3d, ._, ._ }, - .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .cmp, .tmp6d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._d, .mov, .tmp2x, .tmp3d, ._, ._ }, - .{ ._, ._ss, .add, .tmp1x, .tmp2d, ._, ._ }, - .{ .@"1:", ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp5d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .mem(.tmp4d), ._, ._ }, - .{ ._, ._, .cmp, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._ss, .add, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ .@"1:", ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .{ .type = .f64, .kind = .{ .reg = .rcx } }, - .{ .type = .f64, .kind = .{ .reg = .rdx } }, - .{ .type = .f64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_q, .mov, .tmp0q, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .v_q, .mov, .tmp3q, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._, .cmp, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_q, .mov, .src1q, .tmp0q, ._, ._ }, - .{ ._, .v_sd, .add, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .{ .type = .f64, .kind = .{ .reg = .rcx } }, - .{ .type = .f64, .kind = .{ .reg = .rdx } }, - .{ .type = .f64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._q, .mov, .tmp0q, .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._q, .mov, .tmp3q, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._, .cmp, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._q, .mov, .src1x, .tmp0q, ._, ._ }, - .{ ._, ._sd, .add, .dst0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f64, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .{ .type = .f64, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .rdx } }, - .{ .type = .f64, .kind = .{ .reg = .rax } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .movl, .mem(.tmp0q), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._ps, .movl, .mem(.tmp2q), .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .mem(.tmp2q), ._, ._ }, - .{ ._, ._, .cmp, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .mem(.tmp2q), ._, ._, ._ }, - .{ ._, .f_, .add, .mem(.tmp0q), ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2q), ._, ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .movl, .dst0x, .mem(.tmp2q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .{ .type = .f64, .kind = .{ .reg = .rcx } }, - .{ .type = .f64, .kind = .{ .reg = .rdx } }, - .{ .type = .f64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_q, .mov, .tmp3q, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .v_q, .mov, .tmp6q, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .cmp, .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_q, .mov, .tmp2x, .tmp3q, ._, ._ }, - .{ ._, .v_sd, .add, .tmp1x, .tmp1x, .tmp2q, ._ }, - .{ .@"1:", .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .{ .type = .f64, .kind = .{ .reg = .rcx } }, - .{ .type = .f64, .kind = .{ .reg = .rdx } }, - .{ .type = .f64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._q, .mov, .tmp3q, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._q, .mov, .tmp6q, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .cmp, .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._q, .mov, .tmp2x, .tmp3q, ._, ._ }, - .{ ._, ._sd, .add, .tmp1x, .tmp2q, ._, ._ }, - .{ .@"1:", ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, - .{ .type = .f64, .kind = .{ .reg = .rdx } }, - .{ .type = .f64, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .rax } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._ps, .movl, .mem(.tmp5q), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .mem(.tmp5q), ._, ._ }, - .{ ._, ._, .cmp, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .mem(.tmp5q), ._, ._, ._ }, - .{ ._, .f_, .add, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp5q), ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .mem(.tmp5q), ._, ._ }, - .{ .@"1:", ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .avx, .x87, .fast_imm16 }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ }, - .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .avx, .x87, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse2, .x87, .fast_imm16 }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ }, - .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse2, .x87, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse, .x87, .fast_imm16 }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ }, - .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse, .x87, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .avx, .x87, .fast_imm16 }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ }, - .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .avx, .x87, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ }, - .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse2, .x87, .fast_imm16 }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ }, - .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse2, .x87, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ }, - .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse, .x87, .fast_imm16 }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ }, - .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_cc_abi = .sysv64, - .required_features = .{ .@"64bit", .sse, .x87, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, - .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ }, - .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f128, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rcx } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .vp_q, .extr, .tmp3q, .dst0x, .ui(1), ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, .v_q, .mov, .tmp3q, .dst0x, ._, ._ }, - .{ ._, ._, .cmp, .tmp3q, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .src1x, .mem(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse4_1, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f128, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rcx } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .p_q, .extr, .tmp3q, .dst0x, .ui(1), ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._q, .mov, .tmp3q, .dst0x, ._, ._ }, - .{ ._, ._, .cmp, .tmp3q, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._dqa, .mov, .src1x, .mem(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f128, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rcx } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .p_d, .shuf, .src1x, .dst0x, .ui(0b11_10_11_10), ._ }, - .{ ._, ._q, .mov, .tmp3q, .src1x, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._q, .mov, .tmp3q, .dst0x, ._, ._ }, - .{ ._, ._, .cmp, .tmp3q, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._dqa, .mov, .src1x, .mem(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f128, .kind = .mem }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .mem }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .mem(.tmp0x), .src1x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp3x), .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .memd(.tmp3q, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp3q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._ps, .mova, .src1x, .mem(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rcx } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .vp_q, .extr, .tmp5q, .tmp1x, .ui(1), ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, .v_q, .mov, .tmp5q, .tmp1x, ._, ._ }, - .{ ._, ._, .cmp, .tmp5q, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse4_1, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rcx } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .p_q, .extr, .tmp5q, .tmp1x, .ui(1), ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._q, .mov, .tmp5q, .tmp1x, ._, ._ }, - .{ ._, ._, .cmp, .tmp5q, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rcx } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .tmp1x, .ui(0b11_10_11_10), ._ }, - .{ ._, ._q, .mov, .tmp5q, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._q, .mov, .tmp5q, .tmp1x, ._, ._ }, - .{ ._, ._, .cmp, .tmp5q, .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, - .{ .type = .f128, .kind = .{ .reg = .rdx } }, - .{ .type = .f128, .kind = .mem }, - .{ .type = .f128, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp5x), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .memd(.tmp5q, 8), ._, ._ }, - .{ ._, ._, .cmp, .mem(.tmp5q), .si(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, - .{ .@"1:", ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .ptr_add => |air_tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - try ops[0].toSlicePtr(cg); - var res: [1]Temp = undefined; - if (!hack_around_sema_opv_bugs or ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{ - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leaa(.src0, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 2 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"2", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 2 + 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"2", .src1), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 4 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"4", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 4 + 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"4", .src1), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .elem_size_is = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"8", .src1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .elem_size_is = 8 + 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"8", .src1), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .po2_elem_size, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, - } }, - }, .{ - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .add_src0_elem_size), ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - } else res[0] = ops[0]; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .ptr_sub => |air_tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - try ops[0].toSlicePtr(cg); - var res: [1]Temp = undefined; - if (!hack_around_sema_opv_bugs or ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{ - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leaa(.src0, .sub_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .neg, .src1p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 2 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .neg, .src1p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"2", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 2 + 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"2", .src1), ._, ._ }, - .{ ._, ._, .neg, .dst0p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 4 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .neg, .src1p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"4", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .elem_size_is = 4 + 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"4", .src1), ._, ._ }, - .{ ._, ._, .neg, .dst0p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .elem_size_is = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .neg, .src1p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"8", .src1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .elem_size_is = 8 + 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"8", .src1), ._, ._ }, - .{ ._, ._, .neg, .dst0p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .po2_elem_size, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src1 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ }, - .{ ._, ._, .neg, .src1p, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, - } }, - }, .{ - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .sub_src0_elem_size), ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - } else res[0] = ops[0]; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .max => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._l, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._b, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._l, .cmov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._b, .cmov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._l, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._b, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._l, .cmov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._b, .cmov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._l, .cmov, .tmp0p, .tmp1p, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._b, .cmov, .tmp0p, .tmp1p, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .maxs, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .maxs, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._l, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._l, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .maxu, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .maxu, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .maxu, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._b, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._b, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .maxs, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .maxs, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .maxs, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .maxs, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._l, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .maxu, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .maxu, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .subus, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .maxu, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .add, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._b, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .maxs, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .maxs, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .maxs, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._l, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .maxu, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .maxu, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .src1x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .maxu, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp3x, .tmp5x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._b, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpgt, .dst0x, .src1x, .src0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpgt, .dst0y, .src1y, .src0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .tmp1y, ._ }, - .{ ._, .vp_b, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .tmp1x, ._ }, - .{ ._, .vp_b, .blendv, .tmp1x, .tmp1x, .tmp2x, .tmp3x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp1x, .tmp2x, .tmp3x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._l, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .dst0x, .tmp2x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .dst0y, .tmp2y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp6y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp6y, .tmp5y, ._ }, - .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp4y, .tmp5y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp6x, .tmp4x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp6x, .tmp5x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp4x, .tmp5x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_, .xor, .tmp6x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp6x, .tmp5x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp3x, .tmp4x, .tmp6x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._b, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._l, .cmov, .tmp1p, .tmp2p, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._b, .cmov, .tmp1p, .tmp2p, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .cmp, .tmp1x, .dst0d, .dst0x, .vp(.unord) }, - .{ ._, .v_ss, .max, .dst0x, .tmp0d, .dst0x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .dst0x, .dst0x, .vp(.unord) }, - .{ ._, .v_ps, .max, .dst0x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .dst0y, .dst0y, .vp(.unord) }, - .{ ._, .v_ps, .max, .dst0y, .tmp0y, .dst0y, ._ }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, - .{ ._, .v_ps, .max, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .cmp, .tmp0x, .src0x, .src0d, .vp(.unord) }, - .{ ._, .v_ss, .max, .dst0x, .src1x, .src0d, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ss, .max, .dst0x, .src0d, ._, ._ }, - .{ ._, ._ss, .cmp, .src0x, .src0d, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._ss, .max, .tmp0x, .src0d, ._, ._ }, - .{ ._, ._ss, .cmp, .dst0x, .src0d, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, - .{ ._, .v_ps, .max, .dst0x, .src1x, .src0x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .max, .dst0x, .src0x, ._, ._ }, - .{ ._, ._ps, .cmp, .src0x, .src0x, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._ps, .max, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, - .{ ._, .v_ps, .max, .dst0y, .src1y, .src0y, ._ }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, - .{ ._, .v_ps, .max, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .max, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .max, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .tmp1x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .cmp, .tmp0x, .src0x, .src0q, .vp(.unord) }, - .{ ._, .v_sd, .max, .dst0x, .src1x, .src0q, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._sd, .max, .dst0x, .src0q, ._, ._ }, - .{ ._, ._sd, .cmp, .src0x, .src0q, .sp(.unord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._sd, .max, .tmp0x, .src0q, ._, ._ }, - .{ ._, ._sd, .cmp, .dst0x, .src0q, .sp(.ord), ._ }, - .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmax" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, - .{ ._, .v_pd, .max, .dst0x, .src1x, .src0x, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .max, .dst0x, .src0x, ._, ._ }, - .{ ._, ._pd, .cmp, .src0x, .src0x, .sp(.unord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._pd, .max, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._pd, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, - .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, - .{ ._, .v_pd, .max, .dst0y, .src1y, .src0y, ._ }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, - .{ ._, .v_pd, .max, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_pd, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .max, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._pd, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .max, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._pd, .@"and", .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"or", .tmp1x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmax" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp0t, .tmp0t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp0t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._p, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp1t, .tmp1t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp1t, .tmp2t, ._, ._ }, - .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp1t, .tmp2t, ._, ._ }, - .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp1t, .tmp2t, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._p, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._b, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, - .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, - .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .min => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._ge, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._ae, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._ge, .cmov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._ae, .cmov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._ge, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._ae, .cmov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._ge, .cmov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._ae, .cmov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._ge, .cmov, .tmp0p, .tmp1p, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._ae, .cmov, .tmp0p, .tmp1p, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .mins, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .mins, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .mins, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._ge, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._ge, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .minu, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .minu, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .minu, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .minu, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._ae, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._ae, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .mins, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .mins, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .mins, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .mins, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ge, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .minu, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .minu, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .subus, .src0x, .src1x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .minu, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .sub, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ae, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .mins, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .mins, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .mins, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ge, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .minu, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .minu, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .minu, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp4x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp4x, .tmp5x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ae, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpgt, .dst0x, .src0x, .src1x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp0x, .src1x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpgt, .dst0y, .src0y, .src1y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp1y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .tmp1x, .tmp1x, .tmp2x, .tmp3x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp1x, .tmp2x, .tmp3x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ge, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .src1x, ._ }, - .{ ._, .vp_q, .cmpgt, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .src1y, ._ }, - .{ ._, .vp_q, .cmpgt, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp6y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp6y, ._ }, - .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp4y, .tmp5y }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp6x, .tmp4x, .tmp2x, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp6x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp4x, .tmp5x }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp6x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, - .{ ._, .p_, .xor, .tmp6x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, - .{ ._, .p_b, .blendv, .tmp3x, .tmp4x, .tmp5x, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ae, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._ge, .cmov, .tmp1p, .tmp2p, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._ae, .cmov, .tmp1p, .tmp2p, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .reg = .rsi } }, - .{ .type = .u64, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .f16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .cmp, .tmp1x, .dst0x, .dst0d, .vp(.unord) }, - .{ ._, .v_ss, .min, .dst0x, .tmp0x, .dst0d, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .dst0x, .dst0x, .vp(.unord) }, - .{ ._, .v_ps, .min, .dst0x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .dst0y, .dst0y, .vp(.unord) }, - .{ ._, .v_ps, .min, .dst0y, .tmp0y, .dst0y, ._ }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, - .{ ._, .v_ps, .min, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_size), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp4w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .cmp, .tmp0x, .src0x, .src0d, .vp(.unord) }, - .{ ._, .v_ss, .min, .dst0x, .src1x, .src0d, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ss, .min, .dst0x, .src0d, ._, ._ }, - .{ ._, ._ss, .cmp, .src0x, .src0d, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._ss, .min, .tmp0x, .src0d, ._, ._ }, - .{ ._, ._ss, .cmp, .dst0x, .src0d, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, - .{ ._, .v_ps, .min, .dst0x, .src1x, .src0x, ._ }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .min, .dst0x, .src0x, ._, ._ }, - .{ ._, ._ps, .cmp, .src0x, .src0x, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._ps, .min, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, - .{ ._, .v_ps, .min, .dst0y, .src1y, .src0y, ._ }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, - .{ ._, .v_ps, .min, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .min, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .min, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .tmp1x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .cmp, .tmp0x, .src0x, .src0q, .vp(.unord) }, - .{ ._, .v_sd, .min, .dst0x, .src1x, .src0q, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._sd, .min, .dst0x, .src0q, ._, ._ }, - .{ ._, ._sd, .cmp, .src0x, .src0q, .sp(.unord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._sd, .min, .tmp0x, .src0q, ._, ._ }, - .{ ._, ._sd, .cmp, .dst0x, .src0q, .sp(.ord), ._ }, - .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmin" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, - .{ ._, .v_pd, .min, .dst0x, .src1x, .src0x, ._ }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .min, .dst0x, .src0x, ._, ._ }, - .{ ._, ._pd, .cmp, .src0x, .src0x, .sp(.unord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, - .{ ._, ._pd, .min, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._pd, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, - .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, - .{ ._, .v_pd, .min, .dst0y, .src1y, .src0y, ._ }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, - .{ ._, .v_pd, .min, .tmp1y, .tmp2y, .tmp1y, ._ }, - .{ ._, .v_pd, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .min, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._pd, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .min, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._pd, .@"and", .tmp3x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"or", .tmp1x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmin" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp0t, .tmp0t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp0t, .src1t, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp0t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._p, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp1t, .tmp1t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp1t, .tmp2t, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp1t, .tmp2t, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp1t, .tmp2t, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._p, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._b, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, - .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, - .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .add_with_overflow => |air_tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [2]Temp = undefined; - cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .po } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .add, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._rx, .sa, .dst0d, .dst0d, .tmp0d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .add, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._rx, .sa, .dst0q, .dst0q, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._l, .ro, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, - .{ ._, ._, .cmp, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .b } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .add, .src0q, .src1q, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .none }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._o, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_src0_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .dst1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .dst1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .dst1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .dst1q, ._, ._ }, - .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .dst1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .dst1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .dst1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .dst1q, ._, ._ }, - .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_pl.ty.toType().fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].withOverflow(&res[1], cg); - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .sub_with_overflow => |air_tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [2]Temp = undefined; - cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .po } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .add, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._rx, .sa, .dst0d, .dst0d, .tmp0d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .add, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._rx, .sa, .dst0q, .dst0q, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._l, .ro, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .sub, .src0q, .src1q, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .ref = .src1 } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .sub, .src0q, .src1q, ._, ._ }, - .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .none }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._o, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_src0_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .none }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .none }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, - .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .none }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .none }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_pl.ty.toType().fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].withOverflow(&res[1], cg); - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .mul_with_overflow => |air_tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [2]Temp = undefined; - cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nbe } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(16), ._, ._ }, - .{ ._, ._, .mov, .tmp0w, .dst0w, ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mul, .src1w, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .uia(std.math.maxInt(u16), .src0, .sub_umax), ._, ._ }, - .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(32), ._, ._ }, - .{ ._, ._, .@"or", .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._l, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .ref = .src0 } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1d, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"or", .tmp0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0q, .src0q, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .i_, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._l, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .dst0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .ref = .src0 } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .tmp0q, .dst0q, ._ }, - .{ ._, ._, .xor, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .ref = .src0 } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mul, .src1q, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, - .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, - .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, - .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, - .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, - .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, - .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, - .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, - .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, - .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, - .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_pl.ty.toType().fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].withOverflow(&res[1], cg); - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .shl_with_overflow => |air_tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [2]Temp = undefined; - cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0b, .uia(8, .src0, .add_src1_sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._l, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0w, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0w, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .add_src1_sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0w, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .add_src1_sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .add_src1_sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .{ .cc = .nz } }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_pl.ty.toType().fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].withOverflow(&res[1], cg); - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .alloc => { - const ty = air_datas[@intFromEnum(inst)].ty; - const slot = try cg.tempInit(ty, .{ .lea_frame = .{ - .index = try cg.allocMemPtr(inst), - } }); - try slot.finish(inst, &.{}, &.{}, cg); - }, - .inferred_alloc, .inferred_alloc_comptime => unreachable, - .ret_ptr => { - const ty = air_datas[@intFromEnum(inst)].ty; - var slot = switch (cg.ret_mcv.long) { - else => unreachable, - .none => try cg.tempInit(ty, .{ .lea_frame = .{ - .index = try cg.allocMemPtr(inst), - } }), - .load_frame => slot: { - var slot = try cg.tempInit(ty, cg.ret_mcv.long); - try slot.toOffset(cg.ret_mcv.short.indirect.off, cg); - break :slot slot; - }, - }; - try slot.finish(inst, &.{}, &.{}, cg); - }, - .assembly => try cg.airAsm(inst), - .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) { - else => unreachable, - .bit_and, .bool_and => .@"and", - .bit_or, .bool_or => .@"or", - .xor => .xor, - })) { - else => unreachable, - inline .@"and", .@"or", .xor => |mir_tag| comptime &.{ .{ - .src_constraints = .{ .{ .size = .byte }, .{ .size = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .imm8, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .size = .word }, .{ .size = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .size = .dword }, .{ .size = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mm, .to_mm, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_xmm, .to_xmm, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .mem, .none } }, - .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_ymm, .to_ymm, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .mem, .none } }, - .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_ymm, .to_ymm, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .movu, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .movu, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, mir_tag, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, mir_tag, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .shr, .shr_exact => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sa, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sh, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sa, .dst0w, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sh, .dst0w, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sh, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._rx, .sa, .dst0d, .src0d, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._rx, .sh, .dst0d, .src0d, .src1d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sh, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sa, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sh, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._rx, .sa, .dst0q, .src0q, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._rx, .sh, .dst0q, .src0q, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sa, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .sh, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, - .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - cg.typeOf(bin_op.rhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .shl, .shl_exact => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0w, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0w, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._lx, .sh, .dst0d, .src0d, .src1d, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._lx, .sh, .dst0q, .src0q, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .byte }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .unsigned_int = .word }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - cg.typeOf(bin_op.rhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - switch (air_tag) { - else => unreachable, - .shl => res[0].wrapInt(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - res[0].tracking(cg), - }), - else => |e| return e, - }, - .shl_exact => {}, - } - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .shl_sat => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const lhs_ty = cg.typeOf(bin_op.lhs); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - switch (ops[1].tracking(cg).short) { - .immediate => |rhs_imm| res[0] = try cg.tempInit( - .i32, - .{ .immediate = if (rhs_imm < cg.intInfo(lhs_ty).?.bits) - rhs_imm - else - @bitCast(@as(i64, -1)) }, - ), - else => cg.select(&res, &.{.i32}, ops[1..2], comptime &.{ .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .si(1 << 16), ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - .{ ._, ._nb, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .si(1 << 16), ._, ._ }, - .{ ._, ._, .cmc, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .ecx }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(1 << 16), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .mem(.src0q), ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._na, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .ecx }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(1 << 16), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .mem(.src0q), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .sbb, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.rhs).fmt(pt), - ops[1].tracking(cg), - }), - else => |e| return e, - }, - } - if (ops[1].index != res[0].index) try ops[1].die(cg); - ops[1] = res[0]; - cg.select(&res, &.{lhs_ty}, &ops, comptime &.{ .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._l, .sa, .dst0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, - .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, - .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, - .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp1b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0b, .tmp0b, ._, ._ }, - .{ ._, ._na, .cmov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp1b, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0b, .tmp0b, ._, ._ }, - .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .fast_imm16, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .cmp, .tmp1w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._l, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._l, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, - .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, - .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, - .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0w, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .tmp0w, ._, ._ }, - .{ ._, ._na, .cmov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0w, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0w, .tmp0w, ._, ._ }, - .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, - .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .cmp, .tmp1d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._l, .sa, .dst0d, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, - .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0d, .tmp0d, ._, ._ }, - .{ ._, ._na, .cmov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0d, .tmp0d, ._, ._ }, - .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, - .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, - .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .cmp, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._a, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0q, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._l, .sa, .dst0q, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ .@"0:", ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, - .{ ._, ._nz, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, - .{ ._, ._c, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._a, .cmov, .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._z, .cmov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .src0q, ._, ._ }, - .{ ._, ._l, .sh, .tmp1q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - .{ ._, ._na, .cmov, .dst0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .xor, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .src0q, ._, ._ }, - .{ ._, ._l, .sh, .tmp1q, .src1b, ._, ._ }, - .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, - .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .exact_signed_int = 32 }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, - .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, - .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, - .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, - .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, - .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - lhs_ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .not => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .dst0b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .dst0w, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_mm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .mmx }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ }, - .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .dst0q, .lea(.tmp0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_xmm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_xmm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_xmm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .vp(.true) }, - .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ }, - .{ ._, .v_dqu, .mov, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ }, - .{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.true) }, - .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ }, - .{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ }, - .{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.true) }, - .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ .@"0:", .v_, .xor, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memia(.dst0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, 8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, - .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .not, .memad(.dst0d, .add_size, -16), ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ }, - .{ ._, ._, .not, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, - .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .not, .memad(.dst0q, .add_size, -16), ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .not, .memad(.dst0d, .add_size, -8), ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ }, - .{ ._, ._, .not, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, - .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .memad(.dst0d, .add_size, -16), .sa(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .memad(.dst0d, .add_size, -8), .sa(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, - .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ .{ .signed_int_or_full_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_mm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .mmx }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ }, - .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .dst0q, .lea(.tmp0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_xmm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_xmm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_xmm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .vp(.true) }, - .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp2q, .leai(.tmp1q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .leai(.tmp1d, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 4), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .bitcast => try cg.airBitCast(inst), - .block => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const block = cg.air.extraData(Air.Block, ty_pl.payload); - if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none); - try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); - if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_leave_block_none); - }, - .loop => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const block = cg.air.extraData(Air.Block, ty_pl.payload); - try cg.loops.putNoClobber(cg.gpa, inst, .{ - .state = try cg.saveState(), - .target = @intCast(cg.mir_instructions.len), - }); - defer assert(cg.loops.remove(inst)); - try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); - }, - .repeat => { - const repeat = air_datas[@intFromEnum(inst)].repeat; - const loop = cg.loops.get(repeat.loop_inst).?; - try cg.restoreState(loop.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - _ = try cg.asmJmpReloc(loop.target); - }, - .br => try cg.airBr(inst), - .trap => try cg.asmOpOnly(.{ ._2, .ud }), - .breakpoint => try cg.asmOpOnly(.{ ._3, .int }), - .ret_addr => { - var slot = try cg.tempInit(.usize, .{ .load_frame = .{ - .index = .ret_addr, - } }); - while (try slot.toRegClass(true, .general_purpose, cg)) {} - try slot.finish(inst, &.{}, &.{}, cg); - }, - .frame_addr => { - const slot = try cg.tempInit(.usize, .{ .lea_frame = .{ - .index = .base_ptr, - } }); - try slot.finish(inst, &.{}, &.{}, cg); - }, - .call => try cg.airCall(inst, .auto, .{ .safety = true }), - .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }), - .call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }), - .call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }), - .clz => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .si(1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._c, .in, .dst0b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .signed_po2_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .signed_po2_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_po2_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0w, .src0w, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .lzcnt, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, .slow_incdec, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ }, - .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, .slow_incdec, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._c, .st, ._, ._, ._, ._ }, - .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, - .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .tmp3b, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .tmp3b, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .lzcnt, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, - .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, - .{ ._, ._, .neg, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .ctz => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .add, .dst0b, .si(1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._c, .in, .dst0b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi, null, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .tzcnt, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi, .false_deps_lzcnt_tzcnt, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi, null, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi, null, null, null }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ }, - .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .si(16), ._, ._ }, - .{ ._, ._f, .bs, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ }, - .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ }, - .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, - .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ }, - .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, - .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .popcount => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .neg, .dst0b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .popcnt, .dst0d, .dst0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .popcnt, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, .false_deps_popcnt, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0b, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(0x55), ._, ._ }, - .{ ._, ._, .sub, .tmp0b, .src0b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0x33), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0x33), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0x0f), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0b, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(0x55), ._, ._ }, - .{ ._, ._, .sub, .tmp0b, .src0b, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0x33), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0x33), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0x0f), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0w, .ui(0x5555), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ui(0x3333), ._, ._ }, - .{ ._, ._, .@"and", .tmp0w, .ui(0x3333), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0w, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0d, .uia(16, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0w, .ui(0x5555), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ui(0x3333), ._, ._ }, - .{ ._, ._, .@"and", .tmp0w, .ui(0x3333), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0w, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0x5555), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x3333), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x3333), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0d, .uia(16, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0x5555), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x3333), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x3333), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f), ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, - .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0x55555555), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x33333333), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x33333333), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f0f0f), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f0f0f), ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, - .{ ._, .i_, .mul, .dst0d, .tmp0d, .ui(0x01010101), ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(32 - 8), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0x55555555), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x33333333), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x33333333), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f0f0f), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f0f0f), ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, - .{ ._, .i_, .mul, .dst0d, .tmp0d, .ui(0x01010101), ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(32 - 8), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x55555555, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .ui(2), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x33333333, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(4), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x01010101, .none, .repeat), ._, ._ }, - .{ ._, .i_, .mul, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(64 - 8), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sh, .src0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x55555555, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .ui(2), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x33333333, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(4), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0x01010101, .none, .repeat), ._, ._ }, - .{ ._, .i_, .mul, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(64 - 8), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, - .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, - .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ }, - .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ ._, ._l, .sh, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ }, - .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ ._, ._l, .sh, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ }, - .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ }, - .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .byte_swap => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._be, .mov, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .ro, .dst0w, .ui(8), ._, ._ }, - } }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._be, .mov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._be, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .dst0d, ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._be, .mov, .dst0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .dst0d, ._, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._be, .mov, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._be, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .movbe, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._be, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp2q, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp2q, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp2q, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .bit_reverse => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 1 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp1x), .ui(0) }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_b, .broadcast, .tmp1x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_b, .shuf, .tmp1x, .src0x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0b, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0b00110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0b11001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0b01010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0b10101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp1x), .ui(0) }, - .{ ._, .vp_, .unpcklbw, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, .p_w, .sra, .src0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_b, .broadcast, .tmp1x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_b, .shuf, .tmp1x, .src0x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0b, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0b00110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0b11001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0b01010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0b10101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp1x), .ui(0) }, - .{ ._, .vp_, .unpcklbw, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .src0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_b, .broadcast, .tmp1x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_b, .shuf, .tmp1x, .src0x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0b, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0b00110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0b11001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0b, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .ui(0b01010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(0b10101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, - .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.dst0x), ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .lea(.dst0x), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .ro, .src0w, .ui(8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0b0000111100001111), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1111000011110000), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b0011001100110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1100110011001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b0101010101010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1010101010101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, - .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.dst0x), ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp2x, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .lea(.dst0x), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .ro, .src0w, .ui(8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0b0000111100001111), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1111000011110000), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b0011001100110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1100110011001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b0101010101010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1010101010101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - .{ ._, .vp_w, .srl, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .uia(24, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_w, .sll, .tmp2x, .sia(-8, .dst0, .add_bit_size), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.dst0x), ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .lea(.dst0x), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp2x, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .lea(.dst0x), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, - .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, - .{ ._, ._r, .sh, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .ro, .src0w, .ui(8), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0b0000111100001111), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1111000011110000), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b0011001100110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1100110011001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b0101010101010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b1010101010101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b00_10_00_10), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } }, - .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp2y, .src0x, ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2y, .tmp2y, .lea(.dst0y), ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lea(.dst0y), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp2y, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .src0d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0b00001111000011110000111100001111), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b11110000111100001111000011110000), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b00110011001100110011001100110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b11001100110011001100110011001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b01010101010101010101010101010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b10101010101010101010101010101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - .{ ._, .vp_d, .sra, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b00_10_00_10), ._ }, - .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } }, - .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp2y, .src0x, ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2y, .tmp2y, .lea(.dst0y), ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lea(.dst0y), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp2y, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .src0d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0b00001111000011110000111100001111), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b11110000111100001111000011110000), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b00110011001100110011001100110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b11001100110011001100110011001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b01010101010101010101010101010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b10101010101010101010101010101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - .{ ._, .vp_d, .srl, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - .{ ._, .p_d, .srl, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .gfni, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b00_10_00_10), ._ }, - .{ ._, .p_d, .srl, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } }, - .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } }, - .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp2y, .src0x, ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2y, .tmp2y, .lea(.dst0y), ._ }, - .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lea(.dst0y), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_b, .movmsk, .dst0d, .tmp2y, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .src0d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0d, .ui(0b00001111000011110000111100001111), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b11110000111100001111000011110000), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b00110011001100110011001100110011), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b11001100110011001100110011001100), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .ui(0b01010101010101010101010101010101), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ui(0b10101010101010101010101010101010), ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.dst0x), .ui(0), ._ }, - .{ ._, ._q, .mov, .dst0q, .src0x, ._, ._ }, - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .src0q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .gfni, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .tmp1x, .src0x, .lea(.dst0x), .ui(0) }, - .{ ._, .v_q, .mov, .dst0q, .tmp1x, ._, ._ }, - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .gfni, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.dst0x), .ui(0), ._ }, - .{ ._, ._q, .mov, .dst0q, .src0x, ._, ._ }, - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .src0q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - .{ ._, .vp_q, .srl, .dst0x, .dst0x, .uia(64, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - .{ ._, .p_q, .srl, .dst0x, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .gfni, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.dst0x), .ui(0), ._ }, - .{ ._, ._q, .mov, .dst0q, .src0x, ._, ._ }, - .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .b_, .swap, .src0q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - .{ ._, ._r, .sh, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 128 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 128 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .gfni, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .tmp3x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .lea(.tmp2x), ._ }, - .{ ._, .v_q, .mov, .tmp2q, .tmp3x, ._, ._ }, - .{ ._, .vp_q, .extr, .tmp4q, .tmp3x, .ui(1), ._ }, - .{ ._, ._rd, .sh, .tmp2q, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._ }, - .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .ssse3, .gfni, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .lea(.tmp2x), ._, ._ }, - .{ ._, ._q, .mov, .tmp2q, .src0x, ._, ._ }, - .{ ._, .p_q, .extr, .tmp3q, .src0x, .ui(1), ._ }, - .{ ._, ._rd, .sh, .tmp2q, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._ }, - .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .gfni, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .tmp3x, .src0x, .lea(.tmp2x), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .lea(.tmp2x), ._ }, - .{ ._, .v_q, .mov, .tmp2q, .tmp3x, ._, ._ }, - .{ ._, .vp_q, .extr, .tmp4q, .tmp3x, .ui(1), ._ }, - .{ ._, ._rd, .sh, .tmp2q, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._ }, - .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .ssse3, .gfni, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp2x), .ui(0), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .lea(.tmp2x), ._, ._ }, - .{ ._, ._q, .mov, .tmp2q, .src0x, ._, ._ }, - .{ ._, .p_q, .extr, .tmp3q, .src0x, .ui(1), ._ }, - .{ ._, ._rd, .sh, .tmp2q, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._ }, - .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp3q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 256 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0y, .src0y, .lea(.tmp2y), .ui(0) }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp2y), ._ }, - .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b01_00_11_10), ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_int = 256 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .dst0y, .src0y, .lea(.tmp2y), .ui(0) }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp2x), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .v_f128, .insert, .dst0y, .tmp3y, .dst0x, .ui(1) }, - } }, - }, .{ - .required_features = .{ .avx2, .gfni, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .yword, .is = 256 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp5y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ .@"0:", .v_q, .perm, .tmp6y, .lea(.tmp1y), .ui(0b01_00_11_10), ._ }, - .{ ._, .v_b, .gf2p8affineq, .tmp6y, .tmp6y, .tmp4y, .ui(0) }, - .{ ._, .vp_b, .shuf, .tmp6y, .tmp6y, .tmp5y, ._ }, - .{ ._, .v_dqu, .mov, .memi(.dst0y, .tmp0), .tmp6y, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 32), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .gfni, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp5x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp6x, .lea(.tmp1x), ._, ._ }, - .{ ._, .v_b, .gf2p8affineq, .tmp6x, .tmp6x, .tmp4x, .ui(0) }, - .{ ._, .vp_b, .shuf, .tmp6x, .tmp6x, .tmp5x, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp6x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, .b_, .swap, .tmp5q, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, .b_, .swap, .tmp5q, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp5q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .movbe, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, - .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, - .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, - .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .sqrt => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(un_op)}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ss, .sqrt, .dst0x, .dst0x, .dst0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .sqrt, .dst0x, .dst0x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .sqrt, .dst0y, .dst0y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .sqrt, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .sqrt, .dst0x, .dst0x, .src0d, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .sqrt, .dst0x, .src0x, .src0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .sqrt, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .sqrt, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "sqrtf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .sqrt, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .sqrt, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .sqrt, .dst0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .sqrt, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .sqrt, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrtf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrtf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .sqrt, .dst0x, .dst0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .sqrt, .dst0x, .src0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .sqrt, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .sqrt, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .sqrt, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .sqrt, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .sqrt, .dst0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .sqrt, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .sqrt, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .round => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(un_op)}, &ops, switch (air_tag) { - else => unreachable, - inline .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .round => |name| comptime &.{ .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "f" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "f" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "f" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .abs => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, - .{ ._, ._s, .cmov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .si(0), ._, ._ }, - .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, - .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0b, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src0w, ._, ._ }, - .{ ._, ._s, .cmov, .dst0w, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0w, .src0w, ._, ._ }, - .{ ._, ._s, .cmov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .si(0), ._, ._ }, - .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, - .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0d, .src0d, ._, ._ }, - .{ ._, ._s, .cmov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .si(0), ._, ._ }, - .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .sub, .dst0q, .src0q, ._, ._ }, - .{ ._, ._s, .cmov, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .si(0), ._, ._ }, - .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .neg, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sh, .tmp2b, .ui(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp3q, .tmp1q, ._, ._ }, - .{ ._, ._c, .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, .ssse3, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_mm, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .abs, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, .ssse3, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_mm, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .abs, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, .ssse3, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_mm, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .abs, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .abs, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .abs, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .abs, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .abs, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .abs, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .abs, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .abs, .dst0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .abs, .dst0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .abs, .dst0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_b, .abs, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_w, .abs, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_d, .abs, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_b, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_w, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_d, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_b, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_w, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_d, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, .ssse3, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_b, .abs, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, .ssse3, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_w, .abs, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, .ssse3, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .p_d, .abs, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._s, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, - .{ ._, ._s, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._s, .cmov, .tmp1w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .neg, .tmp1d, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._s, .cmov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .neg, .tmp1d, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._s, .cmov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .@"test", .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .neg, .tmp1q, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sh, .tmp3b, .ui(1), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, - .{ ._, ._c, .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .abs, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_ps, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .floor, .ceil, .trunc_float => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(un_op)}, &ops, switch (@as(bits.RoundMode.Direction, switch (air_tag) { - else => unreachable, - .floor => .down, - .ceil => .up, - .trunc_float => .zero, - })) { - else => unreachable, - inline .down, .up, .zero => |direction| comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }) }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorh", - .up => "__ceilh", - .zero => "__trunch", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorh", - .up => "__ceilh", - .zero => "__trunch", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorh", - .up => "__ceilh", - .zero => "__trunch", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorh", - .up => "__ceilh", - .zero => "__trunch", - } } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorh", - .up => "__ceilh", - .zero => "__trunch", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .round, .dst0x, .dst0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .round, .dst0x, .src0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .round, .dst0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .round, .dst0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorf", - .up => "ceilf", - .zero => "truncf", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .round, .dst0y, .src0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .round, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), .rm(.{ - .direction = direction, - .precision = .inexact, - }), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .round, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), .rm(.{ - .direction = direction, - .precision = .inexact, - }), ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorf", - .up => "ceilf", - .zero => "truncf", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorf", - .up => "ceilf", - .zero => "truncf", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .round, .dst0x, .dst0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .round, .dst0x, .src0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }) }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .round, .dst0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .round, .dst0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floor", - .up => "ceil", - .zero => "trunc", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .round, .dst0y, .src0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .round, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), .rm(.{ - .direction = direction, - .precision = .inexact, - }), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .round, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), .rm(.{ - .direction = direction, - .precision = .inexact, - }), ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floor", - .up => "ceil", - .zero => "trunc", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floor", - .up => "ceil", - .zero => "trunc", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floor", - .up => "ceil", - .zero => "trunc", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "__floorx", - .up => "__ceilx", - .zero => "__truncx", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorq", - .up => "ceilq", - .zero => "truncq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorq", - .up => "ceilq", - .zero => "truncq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorq", - .up => "ceilq", - .zero => "truncq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { - else => unreachable, - .down => "floorq", - .up => "ceilq", - .zero => "truncq", - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .neg, .neg_optimized => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(un_op)}, &ops, comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .chs, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_ps, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .cmp_lt, - .cmp_lt_optimized, - .cmp_lte, - .cmp_lte_optimized, - .cmp_gte, - .cmp_gte_optimized, - .cmp_gt, - .cmp_gt_optimized, - => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const cmp_op = air_tag.toCmpOp().?; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - (if (cg.floatBits(cg.typeOf(bin_op.lhs))) |_| err: { - switch (cmp_op) { - else => unreachable, - .lt, .lte => {}, - .gt, .gte => std.mem.swap(Temp, &ops[0], &ops[1]), - } - break :err cg.select(&res, &.{.bool}, &ops, switch (switch (cmp_op) { - else => unreachable, - .lt, .gt => true, - .lte, .gte => false, - }) { - inline false, true => |strict| comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ }, - .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .l, - false => .le, - } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .z, - false => .nc, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - switch (strict) { - true => .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_001), ._, ._ }, - false => .{ ._, ._r, .sh, .tmp2h, .ui(1), ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_x87, .to_x87, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_x87, .to_x87, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .a, - false => .ae, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .z, - false => .nc, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - switch (strict) { - true => .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_001), ._, ._ }, - false => .{ ._, ._r, .sh, .tmp2h, .ui(1), ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_x87, .to_x87, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .z, - false => .nc, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - switch (strict) { - true => .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_001), ._, ._ }, - false => .{ ._, ._r, .sh, .tmp2h, .ui(1), ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (strict) { - true => .l, - false => .le, - } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, - } }, - } }, - }); - } else err: { - res[0] = ops[0].cmpInts(cmp_op, &ops[1], cg) catch |err| break :err err; - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .cmp_eq, - .cmp_eq_optimized, - .cmp_neq, - .cmp_neq_optimized, - => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const cmp_op = air_tag.toCmpOp().?; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - const ty = cg.typeOf(bin_op.lhs); - var res: [1]Temp = undefined; - const OptInfo = struct { - deaths: [2]Air.Inst.Index, - res: [1]Temp, - state: State, - reloc: Mir.Inst.Index, - }; - var opt_info: ?OptInfo = null; - (err: switch (@as(enum { float, int }, if (cg.floatBits(ty)) |_| - .float - else if (cg.intInfo(ty)) |_| - .int - else category: { - const child_ty = ty.optionalChild(zcu); - const has_value_off: u31 = @intCast(child_ty.abiSize(zcu)); - var has_values: [2]Temp = undefined; - opt_info = @as(OptInfo, undefined); - for (&has_values, &ops, &opt_info.?.deaths) |*has_value, *op, *death| { - has_value.* = try op.read(.bool, .{ .disp = has_value_off }, cg); - const child = try op.read(child_ty, .{}, cg); - try op.die(cg); - op.* = child; - death.* = child.index; - } - cg.select( - &opt_info.?.res, - &.{.bool}, - &has_values, - switch (Condition.fromCompareOperatorUnsigned(cmp_op)) { - else => unreachable, - inline .e, .ne => |cc| comptime &.{.{ - .src_constraints = .{ .{ .size = .byte }, .{ .size = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm8, .none } }, - .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - .{ ._, .fromCond(cc), .set, .dst0b, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0b, .src1b, ._, ._ }, - } }, - }}, - }, - ) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - for (has_values) |has_value| for (opt_info.?.res) |opt_res| { - if (has_value.index == opt_res.index) break; - } else try has_value.die(cg); - opt_info.?.state = cg.initRetroactiveState(); - opt_info.?.state.next_temp_index = cg.next_temp_index; - var state = try cg.saveState(); - state.next_temp_index = cg.next_temp_index; - for (ops) |op| try op.die(cg); - try cg.saveRetroactiveState(&opt_info.?.state); - opt_info.?.reloc = try cg.asmJccReloc(.z, undefined); - try cg.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - break :category if (cg.floatBits(child_ty)) |_| .float else .int; - })) { - .float => { - cg.select(&res, &.{.bool}, &ops, switch (switch (air_tag) { - else => unreachable, - .cmp_eq, .cmp_neq => false, - .cmp_eq_optimized, .cmp_neq_optimized => true, - }) { - inline false, true => |optimized| comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ }, - .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z, - true => .nz, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (optimized) { - false => &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2h, .si(0b1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_100), ._, ._ }, - }, - true => &.{ - .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_000), ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z_and_np, - true => .z, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z, - true => .nz, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (optimized) { - false => &.{ - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2h, .si(0b1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_100), ._, ._ }, - }, - true => &.{ - .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_000), ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_x87, .none } }, - .{ .src = .{ .to_x87, .to_x87, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = switch (optimized) { - false => .z, - true => .nz, - } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (optimized) { - false => &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2h, .si(0b1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_100), ._, ._ }, - }, - true => &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_000), ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, - } }, - } }, - }) catch |err| break :err err; - switch (cmp_op) { - else => unreachable, - .eq => {}, - .neq => { - const cc = &res[0].unwrap(cg).temp.tracking(cg).short.eflags; - cc.* = cc.negate(); - }, - } - }, - .int => { - switch (ty.zigTypeTag(zcu)) { - else => {}, - .@"struct", .@"union" => { - assert(ty.containerLayout(zcu) == .@"packed"); - for (&ops) |*op| op.wrapInt(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - op.tracking(cg), - }), - else => |e| return e, - }; - }, - } - res[0] = ops[0].cmpInts(cmp_op, &ops[1], cg) catch |err| break :err err; - }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty.fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - if (opt_info) |*oi| { - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try cg.genCopy(.bool, oi.res[0].tracking(cg).short, res[0].tracking(cg).short, .{}); - try res[0].die(cg); - res[0] = oi.res[0]; - try cg.restoreState(oi.state, &oi.deaths, .{ - .emit_instructions = true, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - cg.performReloc(oi.reloc); - @memset(&ops, res[0]); - } - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .cmp_vector, .cmp_vector_optimized => |air_tag| fallback: { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const vector_cmp = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; - switch (vector_cmp.compareOperator()) { - .eq, .neq => {}, - .lt, .lte, .gte, .gt => if (cg.floatBits(cg.typeOf(vector_cmp.lhs).childType(zcu)) == null) - break :fallback try cg.airCmpVector(inst), - } - var ops = try cg.tempsFromOperands(inst, .{ vector_cmp.lhs, vector_cmp.rhs }); - var res: [1]Temp = undefined; - (err: switch (vector_cmp.compareOperator()) { - .lt, .lte, .gte, .gt => |cmp_op| { - switch (cmp_op) { - else => unreachable, - .lt, .lte => {}, - .gt, .gte => std.mem.swap(Temp, &ops[0], &ops[1]), - } - break :err cg.select(&res, &.{ty_pl.ty.toType()}, &ops, switch (@as(Condition, switch (cmp_op) { - else => unreachable, - .lt, .gt => .l, - .lte, .gte => .le, - })) { - else => unreachable, - inline .l, .le => |cc| comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .tmp0d, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .sse, .mem, .none } }, - .{ .src = .{ .mem, .sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) { - else => unreachable, - .l => .gt, - .le => .ge, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { - else => unreachable, - .l => .gt, - .le => .ge, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { - else => unreachable, - .l => .gt, - .le => .ge, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) { - else => unreachable, - .l => .gt, - .le => .ge, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { - else => unreachable, - .l => .gt, - .le => .ge, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { - else => unreachable, - .l => .gt, - .le => .ge, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - } }, - }, .{ - .required_features = .{ .f16c, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, .p_w, .ackssd, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp4d, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, -16), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ .@"1:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }) }, - .{ ._, .v_pd, .movmsk, .tmp4d, .tmp3y, ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .cmp, .tmp3x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .l => .lt, - .le => .le, - }), ._ }, - .{ ._, ._pd, .movmsk, .tmp4d, .tmp3x, ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, - .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, - .{ ._, .fromCond(switch (cc) { - else => unreachable, - .l => .a, - .le => .ae, - }), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, - .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, - .{ ._, .fromCond(switch (cc) { - else => unreachable, - .l => .a, - .le => .ae, - }), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, - switch (cc) { - else => unreachable, - .l => .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_001), ._, ._ }, - .le => .{ ._, ._r, .sh, .tmp6h, .ui(1), ._, ._ }, - }, - .{ ._, .fromCond(switch (cc) { - else => unreachable, - .l => .z, - .le => .nc, - }), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, - .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, - .{ ._, .fromCond(switch (cc) { - else => unreachable, - .l => .a, - .le => .ae, - }), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, - .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, - .{ ._, .fromCond(switch (cc) { - else => unreachable, - .l => .a, - .le => .ae, - }), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, - switch (cc) { - else => unreachable, - .l => .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_001), ._, ._ }, - .le => .{ ._, ._r, .sh, .tmp6h, .ui(1), ._, ._ }, - }, - .{ ._, .fromCond(switch (cc) { - else => unreachable, - .l => .z, - .le => .nc, - }), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - } }, - }); - }, - .eq, .neq => |cmp_op| cg.select(&res, &.{ty_pl.ty.toType()}, &ops, switch (@as(Condition, switch (cmp_op) { - else => unreachable, - .eq => .e, - .neq => .ne, - })) { - else => unreachable, - inline .e, .ne => |cc| comptime &.{ .{ - .src_constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm8, .none } }, - .{ .src = .{ .imm8, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ }, - .{ ._, ._, .not, .dst0b, ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ }, - }, - } }, - }, .{ - .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ }, - .{ ._, ._, .not, .dst0w, ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ }, - }, - } }, - }, .{ - .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ }, - .{ ._, ._, .not, .dst0d, ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ }, - .{ ._, ._, .not, .dst0q, ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ }, - }, - } }, - }, .{ - .src_constraints = .{ .any_bool_vec, .any_bool_vec, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .not, .tmp1p, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .byte, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .word, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .dword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .qword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .{ .scalar_int = .{ .of = .xword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .byte, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .{ .scalar_int = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .word, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .{ .scalar_int = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .dword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .{ .scalar_int = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .qword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .qword, .is = .byte } }, - .{ .scalar_int = .{ .of = .qword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .byte, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .qword, .is = .word } }, - .{ .scalar_int = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .word, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .mmx, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .qword, .is = .dword } }, - .{ .scalar_int = .{ .of = .qword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mmx, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .dword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .cmpeq, .dst0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .byte } }, - .{ .scalar_int = .{ .of = .yword, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .byte, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .{ .scalar_int = .{ .of = .yword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .word, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .{ .scalar_int = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .dword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .scalar_int = .{ .of = .yword, .is = .qword } }, - .{ .scalar_int = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ - .kind = .all, - .inverted = switch (cc) { - else => unreachable, - .e => false, - .ne => true, - }, - .scalar = .qword, - } } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, - .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp4x, ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ }, - .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, - .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, - .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp4b, .si(0b1111), ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = switch (cc) { - else => unreachable, - .e => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - .ne => &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, - .{ ._, ._, .xor, .tmp4b, .si(0b11), ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any_scalar_int, .any }, - .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, - .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any_scalar_int, .any }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, - .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any_scalar_int, .any }, - .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, - .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, - .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ }, - .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, - .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp4w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp4w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, - .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp4d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, - .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .cmp, .tmp4q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, - .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .tmp0d, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .sse, .mem, .none } }, - .{ .src = .{ .mem, .sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .dword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .ref_mask = .{ - .ref = .src0, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .none } }, - .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .none } }, - }, - .dst_temps = .{ .{ .mut_rc_mask = .{ - .ref = .src0, - .rc = .sse, - .info = .{ .kind = .all, .scalar = .qword }, - } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - } }, - }, .{ - .required_features = .{ .f16c, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, - .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, .p_w, .ackssd, .tmp2x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp4d, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, -16), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, - .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ .@"1:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, - .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, - .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, - .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, - .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }) }, - .{ ._, .v_pd, .movmsk, .tmp4d, .tmp3y, ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._pd, .cmp, .tmp3x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { - else => unreachable, - .e => .eq, - .ne => .neq, - }), ._ }, - .{ ._, ._pd, .movmsk, .tmp4d, .tmp3x, ._, ._ }, - .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, - .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, switch (cc) { - else => unreachable, - .e => ._np, - .ne => ._p, - }, .set, .tmp4b, ._, ._, ._ }, - .{ ._, ._, switch (cc) { - else => unreachable, - .e => .@"and", - .ne => .@"or", - }, .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, - .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, switch (cc) { - else => unreachable, - .e => ._np, - .ne => ._p, - }, .set, .tmp4b, ._, ._, ._ }, - .{ ._, ._, switch (cc) { - else => unreachable, - .e => .@"and", - .ne => .@"or", - }, .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6h, .si(0b1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_100), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, - .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, switch (cc) { - else => unreachable, - .e => ._np, - .ne => ._p, - }, .set, .tmp4b, ._, ._, ._ }, - .{ ._, ._, switch (cc) { - else => unreachable, - .e => .@"and", - .ne => .@"or", - }, .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, - .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, switch (cc) { - else => unreachable, - .e => ._np, - .ne => ._p, - }, .set, .tmp4b, ._, ._, ._ }, - .{ ._, ._, switch (cc) { - else => unreachable, - .e => .@"and", - .ne => .@"or", - }, .tmp3b, .tmp4b, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rcx } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, - .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp6h, .si(0b1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_100), ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, - .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, - .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .u8, .kind = .{ .reg = .cl } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, - .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, - .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, - .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, - .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, - .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, - } }, - } }, - }), - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {s} {f} {f} {f}", .{ - @tagName(air_tag), - @tagName(vector_cmp.compareOperator()), - cg.typeOf(vector_cmp.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ vector_cmp.lhs, vector_cmp.rhs }, &ops, cg); - }, - .cond_br => try cg.airCondBr(inst), - .switch_br => try cg.airSwitchBr(inst), - .loop_switch_br => try cg.airLoopSwitchBr(inst), - .switch_dispatch => try cg.airSwitchDispatch(inst), - .@"try", .try_cold => try cg.airTry(inst), - .try_ptr, .try_ptr_cold => try cg.airTryPtr(inst), - .dbg_stmt => if (!cg.mod.strip) { - const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt; - _ = try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_line_stmt_line_column, - .data = .{ .line_column = .{ - .line = dbg_stmt.line, - .column = dbg_stmt.column, - } }, - }); - }, - .dbg_empty_stmt => if (!cg.mod.strip) { - if (cg.mir_instructions.len > 0) { - const prev_mir_op = &cg.mir_instructions.items(.ops)[cg.mir_instructions.len - 1]; - if (prev_mir_op.* == .pseudo_dbg_line_line_column) - prev_mir_op.* = .pseudo_dbg_line_stmt_line_column; - } - try cg.asmOpOnly(.{ ._, .nop }); - }, - .dbg_inline_block => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const dbg_inline_block = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); - const old_inline_func = cg.inline_func; - defer cg.inline_func = old_inline_func; - cg.inline_func = dbg_inline_block.data.func; - if (!cg.mod.strip) _ = try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_enter_inline_func, - .data = .{ .ip_index = dbg_inline_block.data.func }, - }); - try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[dbg_inline_block.end..][0..dbg_inline_block.data.body_len])); - if (!cg.mod.strip) _ = try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_leave_inline_func, - .data = .{ .ip_index = old_inline_func }, - }); - }, - .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => |air_tag| if (!cg.mod.strip) { - const pl_op = air_datas[@intFromEnum(inst)].pl_op; - const air_name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); - const op_ty = cg.typeOf(pl_op.operand); - const local_ty = switch (air_tag) { - else => unreachable, - .dbg_var_ptr => op_ty.childType(zcu), - .dbg_var_val, .dbg_arg_inline => op_ty, - }; - var ops = try cg.tempsFromOperands(inst, .{pl_op.operand}); - var mcv = ops[0].tracking(cg).short; - switch (mcv) { - else => {}, - .eflags => |cc| switch (cc) { - else => {}, - // These values would self destruct. Maybe we make them use their - // turing complete dwarf expression interpreters for once? - .z_and_np, .nz_or_p => { - try cg.spillEflagsIfOccupied(); - mcv = ops[0].tracking(cg).short; - }, - }, - } - - try cg.mir_locals.append(cg.gpa, .{ - .name = switch (air_name) { - .none => switch (air_tag) { - else => unreachable, - .dbg_arg_inline => .none, - }, - else => try cg.addString(air_name.toSlice(cg.air)), - }, - .type = local_ty.toIntern(), - }); - - try cg.genLocalDebugInfo(air_tag, local_ty, ops[0].tracking(cg).short); - try ops[0].die(cg); - }, - .is_null => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const opt_ty = cg.typeOf(un_op); - const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); - const opt_child_ty = opt_ty.optionalChild(zcu); - const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - while (try ops[0].toBase(false, cg)) {} - try cg.asmMemoryImmediate( - .{ ._, .cmp }, - try ops[0].tracking(cg).short.mem(cg, .{ - .size = if (!opt_repr_is_pl) - .byte - else if (opt_child_ty.isSlice(zcu)) - .ptr - else - .fromSize(opt_child_abi_size), - .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, - }), - .u(0), - ); - const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); - try is_null.finish(inst, &.{un_op}, &ops, cg); - }, - .is_non_null => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const opt_ty = cg.typeOf(un_op); - const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); - const opt_child_ty = opt_ty.optionalChild(zcu); - const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - while (try ops[0].toBase(false, cg)) {} - try cg.asmMemoryImmediate( - .{ ._, .cmp }, - try ops[0].tracking(cg).short.mem(cg, .{ - .size = if (!opt_repr_is_pl) - .byte - else if (opt_child_ty.isSlice(zcu)) - .ptr - else - .fromSize(opt_child_abi_size), - .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, - }), - .u(0), - ); - const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); - try is_non_null.finish(inst, &.{un_op}, &ops, cg); - }, - .is_null_ptr => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const opt_ty = cg.typeOf(un_op).childType(zcu); - const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); - const opt_child_ty = opt_ty.optionalChild(zcu); - const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); - while (try ops[0].toLea(cg)) {} - try cg.asmMemoryImmediate( - .{ ._, .cmp }, - try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) - .byte - else if (opt_child_ty.isSlice(zcu)) - .ptr - else - .fromSize(opt_child_abi_size) }), - .u(0), - ); - const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); - try is_null.finish(inst, &.{un_op}, &ops, cg); - }, - .is_non_null_ptr => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const opt_ty = cg.typeOf(un_op).childType(zcu); - const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); - const opt_child_ty = opt_ty.optionalChild(zcu); - const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); - while (try ops[0].toLea(cg)) {} - try cg.asmMemoryImmediate( - .{ ._, .cmp }, - try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) - .byte - else if (opt_child_ty.isSlice(zcu)) - .ptr - else - .fromSize(opt_child_abi_size) }), - .u(0), - ); - const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); - try is_non_null.finish(inst, &.{un_op}, &ops, cg); - }, - .is_err => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const eu_ty = cg.typeOf(un_op); - const eu_err_ty = eu_ty.errorUnionSet(zcu); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - while (try ops[0].toBase(false, cg)) {} - try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ - .size = cg.memSize(eu_err_ty), - .disp = eu_err_off, - }), .u(0)); - const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); - try is_err.finish(inst, &.{un_op}, &ops, cg); - }, - .is_non_err => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const eu_ty = cg.typeOf(un_op); - const eu_err_ty = eu_ty.errorUnionSet(zcu); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - while (try ops[0].toBase(false, cg)) {} - try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ - .size = cg.memSize(eu_err_ty), - .disp = eu_err_off, - }), .u(0)); - const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); - try is_non_err.finish(inst, &.{un_op}, &ops, cg); - }, - .is_err_ptr => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const eu_ty = cg.typeOf(un_op).childType(zcu); - const eu_err_ty = eu_ty.errorUnionSet(zcu); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - try ops[0].toOffset(eu_err_off, cg); - while (try ops[0].toLea(cg)) {} - try cg.asmMemoryImmediate( - .{ ._, .cmp }, - try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), - .u(0), - ); - const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); - try is_err.finish(inst, &.{un_op}, &ops, cg); - }, - .is_non_err_ptr => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - const eu_ty = cg.typeOf(un_op).childType(zcu); - const eu_err_ty = eu_ty.errorUnionSet(zcu); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - try cg.spillEflagsIfOccupied(); - var ops = try cg.tempsFromOperands(inst, .{un_op}); - try ops[0].toOffset(eu_err_off, cg); - while (try ops[0].toLea(cg)) {} - try cg.asmMemoryImmediate( - .{ ._, .cmp }, - try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), - .u(0), - ); - const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); - try is_non_err.finish(inst, &.{un_op}, &ops, cg); - }, - .load => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const val_ty = ty_op.ty.toType(); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{val_ty}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec_elem, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => res[0] = try ops[0].load(val_ty, .{ - .disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) { - .none => 0, - .runtime => unreachable, - else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), - }, - }, cg), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .ret => try cg.airRet(inst, false), - .ret_safe => try cg.airRet(inst, true), - .ret_load => try cg.airRetLoad(inst), - .store, .store_safe => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - cg.select(&.{}, &.{}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._r, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._s, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .leaa(.src0d, .add_vector_index_div_8_down_4), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .leaa(.src0d, .add_vector_index_div_8_down_4), .tmp0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._s, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => try ops[0].store(&ops[1], .{ - .disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) { - .none => 0, - .runtime => unreachable, - else => |vector_index| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)), - }, - .safe = switch (air_tag) { - else => unreachable, - .store => false, - .store_safe => true, - }, - }, cg), - else => |e| return e, - }; - for (ops) |op| try op.die(cg); - }, - .unreach => {}, - .fptrunc => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvtps2ph, .dst0q, .src0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvtps2ph, .dst0q, .src0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvtps2ph, .dst0x, .src0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp3d), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .mem(.tmp3d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp3d), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .mem(.tmp3d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsd2, .dst0x, .dst0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .cvtsd2, .dst0x, .src0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsd2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtpd2, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .cvtpd2, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtpd2, .dst0x, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtpd2, .tmp1x, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .cvtpd2, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp0x), .dst0x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp4d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp5d, .mem(.tmp4d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp5w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0q, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp3d), .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .mem(.tmp3d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm1 }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .fpext => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_sd, .cvtss2, .dst0x, .dst0x, .dst0d, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_pd, .cvtps2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_pd, .cvtps2, .dst0y, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .zword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0y, .src0x, ._, ._ }, - .{ ._, .v_pd, .cvtps2, .tmp1y, .tmp0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_pd, .mova, .mem(.dst0y), .tmp1y, ._, ._ }, - .{ ._, .v_pd, .cvtps2, .tmp1y, .tmp0x, ._, ._ }, - .{ ._, .v_pd, .mova, .memd(.dst0y, 32), .tmp1y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_pd, .cvtps2, .tmp2y, .tmp1x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp1y, .ui(1), ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp2y, ._, ._ }, - .{ ._, .v_pd, .cvtps2, .tmp2y, .tmp1x, ._, ._ }, - .{ ._, .v_pd, .mova, .memsid(.dst0y, .@"4", .tmp0, 32), .tmp2y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f32, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_ss, .mov, .mem(.tmp1d), .tmp0x, ._, ._ }, - .{ ._, .f_, .ld, .mem(.tmp1d), ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .xmm1 }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, - .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp3d), .tmp2d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp3d), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtss2, .dst0x, .dst0x, .src0d, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .cvtss2, .dst0x, .src0x, .src0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtss2, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0d, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cvtps2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .cvtps2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cvtps2, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .cvtps2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .cvtps2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0d, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0q, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0d, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0d, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp0x), .dst0x, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .intcast => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const dst_ty = ty_op.ty.toType(); - const src_ty = cg.typeOf(ty_op.operand); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{dst_ty}, &ops, if (dst_ty.scalarType(zcu).abiSize(zcu) <= src_ty.scalarType(zcu).abiSize(zcu)) comptime &.{ .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0b, .mem(.src0b), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0w, .mem(.src0w), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .mem(.src0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b10_00_10_00), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b10_00_10_00), ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .yword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } } else comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0q, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0q, .src0b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .usize, .kind = .{ .reg = .edi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sd, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .eax } }, - .{ .type = .usize, .kind = .{ .reg = .edi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sd, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, - .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0q, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0q, .src0w, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .usize, .kind = .{ .reg = .edi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sd, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .eax } }, - .{ .type = .usize, .kind = .{ .reg = .edi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sd, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, - .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._d, .movsx, .dst0q, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._d, .movsx, .tmp0q, .src0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .usize, .kind = .{ .reg = .edi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sd, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .eax } }, - .{ .type = .usize, .kind = .{ .reg = .edi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sd, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, - .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._sq, .lod, ._, ._, ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_d, .movsxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_d, .movzxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxb, .dst0x, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxb, .dst0x, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movsxb, .dst0x, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movzxb, .dst0x, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxb, .dst0y, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxb, .dst0y, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movsxb, .tmp1y, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"8", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movzxb, .tmp1y, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"8", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movsxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movzxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_q, .movsxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_q, .movzxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxb, .dst0x, .src0w, ._, ._ }, - .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxb, .dst0x, .src0w, ._, ._ }, - .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movsxb, .dst0x, .src0w, ._, ._ }, - .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movzxb, .dst0x, .src0w, ._, ._ }, - .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_d, .movsxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_d, .movzxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxw, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxw, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movsxw, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movzxw, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxw, .dst0y, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxw, .dst0y, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movsxw, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movzxw, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movsxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movzxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_q, .movsxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_q, .movzxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxw, .dst0x, .src0d, ._, ._ }, - .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxw, .dst0x, .src0d, ._, ._ }, - .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movsxw, .dst0x, .src0d, ._, ._ }, - .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movzxw, .dst0x, .src0d, ._, ._ }, - .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxd, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxd, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movsxd, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movzxd, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxd, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxd, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movsxd, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movzxd, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movsxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_q, .movzxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_q, .movsxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .p_q, .movzxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._d, .movsx, .tmp1q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movsxd, .dst0x, .src0q, ._, ._ }, - .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .movzxd, .dst0x, .src0q, ._, ._ }, - .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movsxd, .dst0x, .src0q, ._, ._ }, - .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_q, .movzxd, .dst0x, .src0q, ._, ._ }, - .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._d, .movsx, .tmp2q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._sq, .lod, ._, ._, ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .any_scalar_signed_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._sq, .lod, ._, ._, ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .any_scalar_int, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - dst_ty.fmt(pt), - src_ty.fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .intcast_safe => unreachable, - .trunc => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .exact_signed_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0d, .si(1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .exact_signed_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mem(.src0b), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .si(1), ._, ._ }, - .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .exact_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .exact_signed_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .mem(.src0b), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 8 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mem(.src0b), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0b, .sa(.dst0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_int, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mem(.src0b), ._, ._ }, - .{ ._, ._, .@"and", .dst0b, .sa(.dst0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .exact_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .exact_signed_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .mem(.src0w), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 16 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mem(.src0w), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0w, .sa(.dst0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mem(.src0w), ._, ._ }, - .{ ._, ._, .@"and", .dst0w, .sa(.dst0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .exact_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .any_int, .any, .any }, - .dst_constraints = .{ .{ .exact_int = 32 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, - .{ ._, ._l, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0d, .sa(.dst0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, - .{ ._, ._, .@"and", .dst0d, .sa(.dst0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .dst_constraints = .{ .{ .exact_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .mem(.src0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .mem(.src0q), ._, ._ }, - .{ ._, ._l, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .src0q, .dst0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .mem(.src0q), .dst0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .mem(.src0q), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .dst_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_dst0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_dst0_size, -16), ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_signed_int, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_dst0_size, -8), ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .bzhi, .tmp2q, .memad(.src0q, .add_dst0_size, -16), .tmp2q, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp2q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .bzhi, .tmp2q, .memad(.src0q, .add_dst0_size, -8), .tmp2q, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .memad(.src0q, .add_dst0_size, -16), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .memad(.src0q, .add_dst0_size, -8), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 1 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 1 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .si(1), ._, ._ }, - .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._l, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._l, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .@"and", .tmp2b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .@"and", .tmp2b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .sll, .dst0x, .src0x, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .sll, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .sll, .dst0y, .src0y, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .dst0y, .dst0y, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sll, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sll, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .sll, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_w, .sra, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .@"and", .tmp2w, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .sll, .dst0x, .src0x, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .sll, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .sll, .dst0y, .src0y, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .dst0y, .dst0y, .uia(32, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sll, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sll, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .sll, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_d, .sra, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._l, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp3d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, - .{ ._, ._, .@"and", .tmp2d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_q, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), .tmp1q, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, - .{ ._, ._l, .sa, .tmp2q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp3q, .memi(.src0q, .tmp1), .tmp2q, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._l, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._l, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, - .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._l, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_signed_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._l, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp4q, .lea(.tmp1q), .tmp3q, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp4q, .lea(.tmp1q), .tmp3q, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp5q, .lea(.tmp1q), .tmp3q, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .bzhi, .tmp5q, .lea(.tmp1q), .tmp3q, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp3q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, - .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp4q, .lea(.tmp1q), ._, ._ }, - .{ ._, ._sq, .sto, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .optional_payload => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - const pl = if (!hack_around_sema_opv_bugs or ty_op.ty.toType().hasRuntimeBitsIgnoreComptime(zcu)) - try ops[0].read(ty_op.ty.toType(), .{}, cg) - else - try cg.tempInit(ty_op.ty.toType(), .none); - try pl.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .optional_payload_ptr => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .optional_payload_ptr_set => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - if (!opt_ty.optionalReprIsPayload(zcu)) { - const opt_pl_ty = opt_ty.optionalChild(zcu); - const opt_pl_abi_size: i32 = @intCast(opt_pl_ty.abiSize(zcu)); - try ops[0].toOffset(opt_pl_abi_size, cg); - var has_value = try cg.tempInit(.bool, .{ .immediate = 1 }); - try ops[0].store(&has_value, .{}, cg); - try has_value.die(cg); - try ops[0].toOffset(-opt_pl_abi_size, cg); - } - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .wrap_optional => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const opt_ty = ty_op.ty.toType(); - const opt_pl_ty = cg.typeOf(ty_op.operand); - const opt_pl_abi_size: u31 = @intCast(opt_pl_ty.abiSize(zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var opt = try cg.tempAlloc(opt_ty); - try opt.write(&ops[0], .{}, cg); - if (!opt_ty.optionalReprIsPayload(zcu)) { - var has_value = try cg.tempInit(.bool, .{ .immediate = 1 }); - try opt.write(&has_value, .{ .disp = opt_pl_abi_size }, cg); - try has_value.die(cg); - } - try opt.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .unwrap_errunion_payload => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_pl_ty = ty_op.ty.toType(); - const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - const pl = if (!hack_around_sema_opv_bugs or eu_pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) - try ops[0].read(eu_pl_ty, .{ .disp = eu_pl_off }, cg) - else - try cg.tempInit(eu_pl_ty, .none); - try pl.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .unwrap_errunion_err => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_ty = cg.typeOf(ty_op.operand); - const eu_err_ty = ty_op.ty.toType(); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - const err = try ops[0].read(eu_err_ty, .{ .disp = eu_err_off }, cg); - try err.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .unwrap_errunion_payload_ptr => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toOffset(eu_pl_off, cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .unwrap_errunion_err_ptr => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); - const eu_err_ty = ty_op.ty.toType(); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toOffset(eu_err_off, cg); - const err = try ops[0].load(eu_err_ty, .{}, cg); - try err.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .errunion_payload_ptr_set => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); - const eu_err_ty = eu_ty.errorUnionSet(zcu); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toOffset(eu_err_off, cg); - var err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 }); - try ops[0].store(&err, .{}, cg); - try err.die(cg); - try ops[0].toOffset(eu_pl_off - eu_err_off, cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .wrap_errunion_payload => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_ty = ty_op.ty.toType(); - const eu_err_ty = eu_ty.errorUnionSet(zcu); - const eu_pl_ty = cg.typeOf(ty_op.operand); - const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - const eu_pl_off: u31 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var eu = try cg.tempAlloc(eu_ty); - try eu.write(&ops[0], .{ .disp = eu_pl_off }, cg); - var err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 }); - try eu.write(&err, .{ .disp = eu_err_off }, cg); - try err.die(cg); - try eu.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .wrap_errunion_err => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const eu_ty = ty_op.ty.toType(); - const eu_pl_ty = eu_ty.errorUnionPayload(zcu); - const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var eu = try cg.tempAlloc(eu_ty); - try eu.write(&ops[0], .{ .disp = eu_err_off }, cg); - try eu.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .struct_field_ptr => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{struct_field.struct_operand}); - try ops[0].toOffset(@intCast(codegen.fieldOffset( - cg.typeOf(struct_field.struct_operand), - ty_pl.ty.toType(), - struct_field.field_index, - zcu, - )), cg); - try ops[0].finish(inst, &.{struct_field.struct_operand}, &ops, cg); - }, - .struct_field_ptr_index_0, - .struct_field_ptr_index_1, - .struct_field_ptr_index_2, - .struct_field_ptr_index_3, - => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toOffset(@intCast(codegen.fieldOffset( - cg.typeOf(ty_op.operand), - ty_op.ty.toType(), - switch (air_tag) { - else => unreachable, - .struct_field_ptr_index_0 => 0, - .struct_field_ptr_index_1 => 1, - .struct_field_ptr_index_2 => 2, - .struct_field_ptr_index_3 => 3, - }, - zcu, - )), cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .struct_field_val => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; - const agg_ty = cg.typeOf(struct_field.struct_operand); - const field_ty = ty_pl.ty.toType(); - const field_off: u31 = switch (agg_ty.containerLayout(zcu)) { - .auto, .@"extern" => @intCast(agg_ty.structFieldOffset(struct_field.field_index, zcu)), - .@"packed" => unreachable, - }; - var ops = try cg.tempsFromOperands(inst, .{struct_field.struct_operand}); - var res = if (!hack_around_sema_opv_bugs or field_ty.hasRuntimeBitsIgnoreComptime(zcu)) - try ops[0].read(field_ty, .{ .disp = field_off }, cg) - else - try cg.tempInit(field_ty, .none); - try res.finish(inst, &.{struct_field.struct_operand}, &ops, cg); - }, - .set_union_tag => { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const union_ty = cg.typeOf(bin_op.lhs).childType(zcu); - const union_layout = union_ty.unionGetLayout(zcu); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - if (!hack_around_sema_opv_bugs or union_layout.tag_size > 0) try ops[0].store(&ops[1], .{ - .disp = @intCast(union_layout.tagOffset()), - }, cg); - const res = try cg.tempInit(.void, .none); - try res.finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .get_union_tag => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const union_ty = cg.typeOf(ty_op.operand); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - const union_layout = union_ty.unionGetLayout(zcu); - assert(union_layout.tag_size > 0); - const res = try ops[0].read(ty_op.ty.toType(), .{ - .disp = @intCast(union_layout.tagOffset()), - }, cg); - try res.finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .slice => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - try ops[0].toPair(&ops[1], cg); - try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .slice_len => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toSliceLen(cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .slice_ptr => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toSlicePtr(cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .ptr_slice_len_ptr => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toOffset(8, cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .ptr_slice_ptr_ptr => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].toOffset(0, cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .array_elem_val => { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const array_ty = cg.typeOf(bin_op.lhs); - const res_ty = array_ty.elemType2(zcu); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - var res: [1]Temp = undefined; - cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm32, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .src0d, .ua(.none, .add_src1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .imm32, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .src0q, .ua(.none, .add_src1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .src0q, .src1q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .imm32, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .mema(.src0d, .add_src1_div_8_down_4), .ua(.none, .add_src1_rem_32), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .c }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .bt, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mema(.src0b, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .memi(.src0b, .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .mema(.src0w, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .memsi(.src0w, .@"2", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .mema(.src0d, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .memsi(.src0d, .@"4", .src1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .mema(.src0q, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .memsi(.src0q, .@"8", .src1), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => { - const elem_size = res_ty.abiSize(zcu); - var base = try cg.tempAllocReg(.usize, abi.RegisterClass.gp); - while (try ops[0].toBase(false, cg) or - try ops[1].toRegClass(true, .general_purpose, cg) or - try base.toRegClass(true, .general_purpose, cg)) - {} - const base_reg = base.tracking(cg).short.register.to64(); - const rhs_reg = ops[1].tracking(cg).short.register.to64(); - if (!std.math.isPowerOfTwo(elem_size)) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - rhs_reg, - rhs_reg, - .u(elem_size), - ); - try cg.asmRegisterMemory( - .{ ._, .lea }, - base_reg, - try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), - ); - } else if (elem_size > 8) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterImmediate( - .{ ._l, .sh }, - rhs_reg, - .u(std.math.log2_int(u64, elem_size)), - ); - try cg.asmRegisterMemory( - .{ ._, .lea }, - base_reg, - try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), - ); - } else try cg.asmRegisterMemory( - .{ ._, .lea }, - base_reg, - try ops[0].tracking(cg).short.mem(cg, .{ - .index = rhs_reg, - .scale = .fromFactor(@intCast(elem_size)), - }), - ); - // Hack around Sema insanity: lhs could be an arbitrarily large comptime-known array - // which could easily get spilled by the upcoming `load`, which would infinite recurse - // since spilling an array requires the same operation that triggered the spill. - try ops[0].die(cg); - ops[0] = base; - res[0] = try ops[0].load(res_ty, .{}, cg); - }, - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .slice_elem_val, .ptr_elem_val => { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - const res_ty = cg.typeOf(bin_op.lhs).elemType2(zcu); - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - try ops[0].toSlicePtr(cg); - var res: [1]Temp = undefined; - if (!hack_around_sema_opv_bugs or res_ty.hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .leaa(.src0b, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .leai(.src0b, .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .leaa(.src0w, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .leasi(.src0w, .@"2", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .leaa(.src0d, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0d, .leasi(.src0d, .@"4", .src1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .leaa(.src0q, .add_src0_elem_size_mul_src1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .dst0q, .leasi(.src0q, .@"8", .src1), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => { - const elem_size = res_ty.abiSize(zcu); - while (true) for (&ops) |*op| { - if (try op.toRegClass(true, .general_purpose, cg)) break; - } else break; - const lhs_reg = ops[0].tracking(cg).short.register.to64(); - const rhs_reg = ops[1].tracking(cg).short.register.to64(); - if (!std.math.isPowerOfTwo(elem_size)) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - rhs_reg, - rhs_reg, - .u(elem_size), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ - .base = .{ .reg = lhs_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else if (elem_size > 8) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterImmediate( - .{ ._l, .sh }, - rhs_reg, - .u(std.math.log2_int(u64, elem_size)), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ - .base = .{ .reg = lhs_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ - .base = .{ .reg = lhs_reg }, - .mod = .{ .rm = .{ - .index = rhs_reg, - .scale = .fromFactor(@intCast(elem_size)), - } }, - }); - res[0] = try ops[0].load(res_ty, .{}, cg); - }, - else => |e| return e, - } else res[0] = try cg.tempInit(res_ty, .none); - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .slice_elem_ptr, .ptr_elem_ptr => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); - try ops[0].toSlicePtr(cg); - const dst_ty = ty_pl.ty.toType(); - if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: { - const elem_size = dst_ty.childType(zcu).abiSize(zcu); - if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset; - while (true) for (&ops) |*op| { - if (try op.toRegClass(true, .general_purpose, cg)) break; - } else break; - const lhs_reg = ops[0].tracking(cg).short.register.to64(); - const rhs_reg = ops[1].tracking(cg).short.register.to64(); - if (!std.math.isPowerOfTwo(elem_size)) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - rhs_reg, - rhs_reg, - .u(elem_size), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ - .base = .{ .reg = lhs_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else if (elem_size > 8) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterImmediate( - .{ ._l, .sh }, - rhs_reg, - .u(std.math.log2_int(u64, elem_size)), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ - .base = .{ .reg = lhs_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ - .base = .{ .reg = lhs_reg }, - .mod = .{ .rm = .{ - .index = rhs_reg, - .scale = .fromFactor(@intCast(elem_size)), - } }, - }); - } - try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); - }, - .array_to_slice => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var len = try cg.tempInit(.usize, .{ - .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), - }); - try ops[0].toPair(&len, cg); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .int_from_float, .int_from_float_optimized => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .dst0q, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .dst0d, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .word }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttps2dq, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_, .cvttps2dq, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttps2dq, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_, .cvttps2dq, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_, .cvttps2dq, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .vp_w, .ackssd, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_, .cvttps2dq, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .vp_w, .ackusd, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_, .cvttps2dq, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp2q, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp2d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp2q, .tmp1x, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp2q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d0, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d0, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp1q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d0, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d0, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp1q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", .v_ps, .xor, .tmp4x, .tmp4x, .tmp4x, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .vp_w, .insr, .tmp4x, .tmp4x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", .v_ps, .xor, .tmp4x, .tmp4x, .tmp4x, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .vp_w, .insr, .tmp4x, .tmp4x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .p_w, .insr, .tmp4x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .p_w, .insr, .tmp4x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixhfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp3d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp5x, .tmp4d, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp3d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp5x, .tmp4d, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttss2si, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttss2si, .dst0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttss2si, .dst0q, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttss2si, .dst0q, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_, .cvttss2si, .tmp2q, .src0d, ._, ._ }, - .{ ._, .v_ss, .sub, .tmp0x, .src0x, .lea(.tmp1d), ._ }, - .{ ._, .v_, .cvttss2si, .dst0q, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._, .cvttss2si, .tmp1q, .src0d, ._, ._ }, - .{ ._, ._ss, .sub, .src0x, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .cvttss2si, .dst0q, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src0x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, - .{ ._, .v_ps, .@"and", .src0x, .src0x, .lea(.tmp1x), ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_d, .mov, .tmp0d, .src0x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, - .{ ._, .v_ps, .@"and", .src0x, .src0x, .lea(.tmp1x), ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_q, .mov, .tmp1q, .src0x, ._, ._ }, - .{ ._, .vp_q, .extr, .tmp4q, .src0x, .ui(1), ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._d, .mov, .tmp0d, .src0x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, - .{ ._, ._ps, .@"and", .src0x, .lea(.tmp1x), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._d, .mov, .tmp0d, .src0x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, - .{ ._, ._ps, .@"and", .src0x, .lea(.tmp1x), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._q, .mov, .tmp1q, .src0x, ._, ._ }, - .{ ._, .p_q, .extr, .tmp4q, .src0x, .ui(1), ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._d, .mov, .tmp0d, .src0x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, - .{ ._, ._ps, .@"and", .src0x, .lea(.tmp1x), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._q, .mov, .tmp1q, .src0x, ._, ._ }, - .{ ._, .p_d, .shuf, .src0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._q, .mov, .tmp4q, .src0x, ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .tmp0x, .src0d, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .tmp0x, .lea(.tmp1x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._d, .movsx, .tmp4q, .src0d, ._, ._ }, - .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp4q, ._, ._ }, - .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .sub, .tmp1q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, - .{ ._, ._, .sbb, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp5q, ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .lea, .tmp6p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .mem(.dst0x), .src0x, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .mem(.dst0x), .src0x, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .mov, .mem(.dst0q), .tmp1q0, ._, ._ }, - .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q1, ._, ._ }, - .{ ._, ._, .xor, .tmp1d0, .tmp1d0, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u32, .kind = .{ .reg = .ecx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .mem(.dst0x), .src0x, ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttps2dq, .tmp3x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttps2dq, .tmp3x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttss2si, .tmp1d, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttss2si, .tmp1d, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, .vp_w, .ackssd, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, .p_w, .ackssd, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, .vp_w, .ackusd, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, .p_w, .ackusd, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_, .cvttss2si, .tmp1q, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttss2si, .tmp1q, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttsd2si, .dst0d, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttsd2si, .dst0d, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, - .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, - .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .fi_p, .st, .dst0w, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .fi_p, .st, .dst0d, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttsd2si, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttsd2si, .dst0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .fi_p, .st, .dst0q, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .@"0x1p63_mem" = .{} } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_, .cvttsd2si, .tmp2q, .src0d, ._, ._ }, - .{ ._, .v_sd, .sub, .tmp0x, .src0x, .lea(.tmp1q), ._ }, - .{ ._, .v_, .cvttsd2si, .dst0q, .tmp0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, - .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp2q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .@"0x1p63_mem" = .{} } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._, .cvttsd2si, .tmp1q, .src0d, ._, ._ }, - .{ ._, ._sd, .sub, .src0x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .cvttsd2si, .dst0q, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .x87, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, - .{ .type = .i64, .kind = .mem }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .dst0b, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttpd2dq, .tmp3x, .memsi(.src0y, .@"8", .tmp0), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttpd2dq, .tmp3x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttpd2dq, .tmp3x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._d, .mov, .tmp4d, .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttsd2si, .tmp1d, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttsd2si, .tmp1d, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .tmp4b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp2b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .tmp4b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp2b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, - .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, - .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttpd2dq, .tmp1x, .memsi(.src0y, .@"4", .tmp0), ._, ._ }, - .{ ._, .vp_w, .ackssd, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttpd2dq, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, .p_w, .ackssd, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._d, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttpd2dq, .tmp1x, .memsi(.src0y, .@"4", .tmp0), ._, ._ }, - .{ ._, .vp_w, .ackusd, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttpd2dq, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, .p_w, .ackusd, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._d, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttsd2si, .tmp1d, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .memi(.dst0w, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .i32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp2w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_, .cvttpd2dq, .tmp1x, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .cvttpd2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._q, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .i64, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_, .cvttsd2si, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .cvttsd2si, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .memi(.dst0q, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .movl, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, - .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, - .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .tmp2b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, .slow_incdec, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .movsx, .tmp4d, .tmp3b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .movsx, .tmp4d, .tmp3b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, - .{ ._, ._, .movsx, .tmp3d, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, - .{ ._, ._, .movsx, .tmp3d, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, .slow_incdec, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .tmp3b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp4d, .tmp3b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i16, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .tmp5b, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fi_p, .stt, .dst0w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .fi_p, .st, .dst0w, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .memi(.dst0w, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .memi(.dst0w, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i32, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp2w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fi_p, .stt, .dst0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .fi_p, .st, .dst0d, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .memi(.dst0d, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i64, .kind = .mem }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp2q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .i64, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fi_p, .stt, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, .fi_p, .st, .dst0q, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .@"64bit", .sse3, .x87, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .i64, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp4q, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp4q, ._, ._ }, - .{ ._, .f_, .sub, .lea(.tmp0d), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, - .{ ._, .fi_p, .stt, .tmp4q, ._, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .@"64bit", .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, - .{ .type = .i64, .kind = .mem }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .dst0b, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .@"64bit", .x87, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_x87, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, - .{ .type = .i64, .kind = .mem }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, - .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .dst0b, ._, ._ }, - .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ }, - .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, - .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .x87, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .stt, .memi(.dst0q, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, .fi_p, .st, .memi(.dst0q, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .src0x, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .size = 16, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, - .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, - .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, - .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .float = .xword }, .any, .any }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .int_from_float_safe => unreachable, - .int_from_float_optimized_safe => unreachable, - .float_from_int => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0q, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .dst0d, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - .{ .@"0:", .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .src0d, .src0b, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .src0d, .src0b, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .src0d, .src0w, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .src0d, .src0w, ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memsi(.dst0x, .@"2", .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memsi(.dst0q, .@"2", .tmp0), .tmp1x, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memsi(.dst0x, .@"2", .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memsi(.dst0q, .@"2", .tmp0), .tmp1x, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtdq2, .dst0x, .src0x, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtdq2, .dst0y, .src0y, ._, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .avx2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtdq2, .tmp1y, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtdq2, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .f16c, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_ss, .cvtsi2, .tmp1x, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._ }, - .{ ._, .v_, .cvtps2ph, .tmp1q, .tmp1x, .rm(.{}), ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse4_1, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse4_1, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d0, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d0, .mem(.tmp4d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp4d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d0, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d0, .mem(.tmp4d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp4d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse4_1, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp2d, .tmp5x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp6d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse4_1, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp2d, .tmp5x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, - .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .mem }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .tmp6d, .tmp5d, ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp6d, ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, - .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .dst0d, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .dst0x, .src0q, ._, ._ }, - .{ ._, ._ss, .add, .dst0x, .dst0d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._ss, .cvtsi2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateisf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneisf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ }, - .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ }, - .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ }, - .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ }, - .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtdq2, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .cvtdq2, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtdq2, .dst0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtdq2, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtdq2, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .cvtdq2, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .cvtsi2, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ps, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_ss, .cvtsi2, .tmp1x, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ss, .cvtsi2, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneisf" } }, - .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .src0d, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i32, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1d, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0d, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .x87, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i64, .kind = .mem }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .u32_0x1p52_hi_0x1p84_hi_0_0_mem }, - .{ .type = .vector_2_f64, .kind = .f64_0x1p52_0x1p84_mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_q, .mov, .tmp0x, .src0q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .unpckldq, .tmp0x, .tmp0x, .lea(.tmp1x), ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_pd, .sub, .tmp0x, .tmp0x, .lea(.tmp1x), ._ }, - .{ ._, .v_pd, .unpckh, .dst0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .u32_0x1p52_hi_0x1p84_hi_0_0_mem }, - .{ .type = .vector_2_f64, .kind = .f64_0x1p52_0x1p84_mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._q, .mov, .tmp0x, .src0q, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .unpckldq, .tmp0x, .lea(.tmp1x), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, - .{ ._, ._pd, .sub, .tmp0x, .lea(.tmp1x), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._pd, .unpckh, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .x87, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, - .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, .fi_, .ld, .dst0q, ._, ._, ._ }, - .{ ._, .f_, .add, .tmp1t, .tmp1t, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .dst0q, .src0q, ._, ._ }, - .{ ._, .fi_, .ld, .dst0q, ._, ._, ._ }, - .{ .@"0:", .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"8", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, .vp_d, .movsxb, .tmp1x, .tmp1d, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, .p_d, .movsxb, .tmp1x, .tmp1d, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, ._pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"8", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, .vp_d, .movzxb, .tmp1x, .tmp1d, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, .p_d, .movzxb, .tmp1x, .tmp1d, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, ._pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .vp_d, .movsxw, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .p_d, .movsxw, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, ._pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .vp_d, .movzxw, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .p_d, .movzxw, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, - .{ ._, ._pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cvtdq2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .cvtdq2, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .cvtdq2, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .cvtdq2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .cvtdq2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .cvtdq2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_pd, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .v_sd, .cvtsi2, .tmp1x, .tmp1x, .memi(.src0q, .tmp0), ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._pd, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._sd, .cvtsi2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, - .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i32, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1d, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0d, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .x87, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .{ .type = .i64, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, .fi_, .ld, .tmp1q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f32, .kind = .f32_0_0x1p64_mem }, - .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .memd(.src0d, 4), ._, ._ }, - .{ ._, ._r, .sh, .tmp3d, .ui(31), ._, ._ }, - .{ ._, .f_, .add, .leasi(.tmp1d, .@"4", .tmp3), ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f32, .kind = .f32_0_0x1p64_mem }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .memd(.src0d, 4), ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(31), ._, ._ }, - .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .mem(.tmp4), ._, ._ }, - .{ ._, .f_, .add, .leasi(.tmp3d, .@"4", .tmp1), ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .mem }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, - .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .movsx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .movsx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, - .{ ._, ._, .movzx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .fi_, .ld, .memi(.src0w, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .movsx, .tmp2d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .fi_, .ld, .memi(.src0d, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .fi_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatdixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatundixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"2", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp1), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp1), ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .mov, .tmp3p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .slow_incdec, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, - .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - cg.typeOf(ty_op.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .reduce => |air_tag| { - const nan = std.math.nan(f16); - const reduce = air_datas[@intFromEnum(inst)].reduce; - const res_ty = cg.typeOfIndex(inst); - var ops = try cg.tempsFromOperands(inst, .{reduce.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{res_ty}, &ops, switch (reduce.operation) { - .And, .Or, .Xor => switch (@as(Mir.Inst.Tag, switch (reduce.operation) { - else => unreachable, - .And => .@"and", - .Or => .@"or", - .Xor => .xor, - })) { - else => unreachable, - inline .@"and", .@"or", .xor => |mir_tag| comptime &switch (mir_tag) { - else => unreachable, - .@"and" => [_]Select.Case{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .@"test", .src0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_, .@"test", .src0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_b, .cmpeq, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"test", .src0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .@"test", .src0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .@"test", .src0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .cmpeq, .tmp0y, .tmp0y, .tmp0y, ._ }, - .{ ._, .vp_, .@"test", .src0y, .tmp0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.true) }, - .{ ._, .v_ps, .@"test", .src0y, .tmp0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .ptest_mask_mem = .src0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"test", .src0y, .lea(.tmp0y), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .si(-1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .src0b, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .si(-1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .src0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .si(-1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .src0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .si(-1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .not, .src0q, ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .src0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nc }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, - .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(-1), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(-8), ._, ._ }, - .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nc }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_2_smin), ._, ._ }, - .{ ._, ._, .@"or", .tmp0q, .mema(.src0q, .add_bit_size_div_8_down_8), ._, ._ }, - .{ ._, ._, .cmp, .tmp0q, .si(-1), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, - .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(-1), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0d, .si(-8), ._, ._ }, - .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, - } }, - } }, - .@"or" => [_]Select.Case{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .@"test", .src0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_, .@"test", .src0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_b, .cmpeq, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"test", .src0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .@"test", .src0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .@"test", .src0y, .src0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .cmpeq, .tmp0y, .tmp0y, .tmp0y, ._ }, - .{ ._, .vp_, .@"test", .src0y, .tmp0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.true) }, - .{ ._, .v_ps, .@"test", .src0y, .tmp0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .any_bool_vec, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .ptest_mask_mem = .src0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"test", .src0y, .lea(.tmp0y), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .ne }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .si(0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .ne }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .si(0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .ne }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .si(0), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .ne }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .src0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nc }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, - .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"1b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nc }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"test", .mema(.src0q, .add_bit_size_div_8_down_8), .tmp0q, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, - .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, - .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"1b", ._, ._, ._ }, - } }, - } }, - .xor => [_]Select.Case{ .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .popcnt, .tmp0w, .src0w, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, .fast_imm16, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0w, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .popcnt, .src0w, .src0w, ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .popcnt, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ua(.src0b, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, .false_deps_popcnt, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .popcnt, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, .false_deps_popcnt, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(16), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .popcnt, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .src0d, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .popcnt, .src0d, .src0d, ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(16), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ua(.src0w, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(16), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ua(.src0w, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .popcnt, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .src0q, .src0q, ._, ._ }, - .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .popcnt, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .ui(32), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ }, - .{ ._, ._, .popcnt, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .ui(32), ._, ._ }, - .{ ._, ._, .@"and", .tmp0d, .ua(.src0d, .add_umax), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp1q, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .popcnt, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .tmp1q, ._, ._ }, - .{ .@"0:", ._, .xor, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1q, .ui(32), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .popcnt, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .mema(.src0q, .add_bit_size_div_8_down_8), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp0q, .memi(.src0q, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .popcnt, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .bool, .any }, - .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .po }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, - .{ ._, ._, .@"and", .tmp0q, .mema(.src0q, .add_bit_size_div_8_down_8), ._, ._ }, - .{ .@"0:", ._, .xor, .tmp0q, .memi(.src0q, .tmp1), ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0q, .ui(32), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, - .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, - } }, - } }, - } ++ [_]Select.Case{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, mir_tag, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, mir_tag, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, mir_tag, .dst0w, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, mir_tag, .dst0d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, mir_tag, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .tmp0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .memd(.src0x, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .tmp0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_2_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .yword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .yword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .yword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .yword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .{ .type = .vector_1_u256, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .yword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .yword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ - .ref = .src0, - .invert = switch (mir_tag) { - else => unreachable, - .@"and" => true, - .@"or", .xor => false, - }, - } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, switch (mir_tag) { - else => unreachable, - .@"and" => .@"or", - .@"or", .xor => .@"and", - }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memi(.src0, .tmp0), ._, ._ }, - .{ .@"1:", ._, .mov, .tmp3q, .leai(.tmp2q, .tmp1), ._, ._ }, - .{ ._, ._, mir_tag, .memi(.dst0q, .tmp1), .tmp3q, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }, - .Min => comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_b, .mins, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_b, .minu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_b, .mins, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_b, .minu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_b, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_b, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_b, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_b, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_b, .mins, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_b, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_b, .mins, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_b, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .vp_b, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_b, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_b, .minu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_b, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_b, .minu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_b, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_b, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mins, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .minu, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .minu, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_w, .sub, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_w, .minposu, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .ph_w, .minposu, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_ps, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp0x, .lea(.tmp1d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_w, .mins, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp1x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_w, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_w, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_w, .mins, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_w, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .mins, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_w, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .vp_w, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._, .movddup, .tmp4x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_w, .minu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_w, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_w, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_d, .mins, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_d, .minu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_d, .mins, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_d, .minu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .mins, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_d, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .mins, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .vp_d, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_d, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .mins, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_d, .minu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_d, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .minu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_d, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_d, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .minu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .v_i128, .extract, .dst0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .v_f128, .extract, .dst0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp1y, .tmp2y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .memad(.src0y, .add_unaligned_size, -32), ._ }, - .{ .@"0:", .vp_, .xor, .tmp3y, .tmp2y, .memi(.src0y, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .memad(.src0x, .add_unaligned_size, -16), ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp2x, .memi(.src0x, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memad(.src0y, .add_size, -32), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -64), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ .@"0:", .vp_, .xor, .tmp4x, .tmp2x, .memi(.src0x, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nl, .cmov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nb, .cmov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._nl, .cmov, .tmp1p, .tmp3p, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._nb, .cmov, .tmp1p, .tmp3p, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp2y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp0y, .tmp0y, .tmp2y, .tmp1y }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, .v_ps, .min, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .min, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .@"or", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .min, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .min, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ .@"0:", .v_ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .min, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._sd, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._sd, .min, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .min, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .min, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_sd, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_pd, .min, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_pd, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_pd, .min, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_sd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._pd, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._sd, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._pd, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._pd, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._sd, .min, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_pd, .min, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ .@"0:", .v_pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._pd, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._sd, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._pd, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._sd, .min, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - .Max => comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_b, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_b, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_b, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_b, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_b, .maxs, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_b, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .vp_b, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_b, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_b, .maxu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_b, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_b, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .signed_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, .slow_incdec, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .maxs, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .maxu, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxu, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .cmpeq, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_ps, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp0x, .lea(.tmp1d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp1x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_w, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ .@"0:", .vp_w, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp1x, .tmp1x, ._, ._ }, - .{ .@"0:", .p_w, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_w, .subus, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_w, .maxs, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_w, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .vp_w, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._, .movddup, .tmp4x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_w, .maxu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_w, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_w, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, - .{ .@"0:", .p_w, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_w, .subus, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .signed_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .src0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .maxs, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_d, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .vp_d, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .p_d, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxs, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, - .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_d, .maxu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_d, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_d, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .maxu, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .signed_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, - .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, - .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .v_i128, .extract, .dst0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .v_f128, .extract, .dst0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"and", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1y, .tmp0y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp1y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp1y, .tmp2y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .dst0y, .tmp2y, .memad(.src0y, .add_unaligned_size, -32), ._ }, - .{ .@"0:", .vp_, .xor, .tmp3y, .tmp2y, .memi(.src0y, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_, .xor, .dst0x, .tmp2x, .memad(.src0x, .add_unaligned_size, -16), ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp2x, .memi(.src0x, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ .@"0:", .v_dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp4y, .memad(.src0y, .add_size, -32), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -64), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"and", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, - .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .xor, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ .@"0:", .vp_, .xor, .tmp4x, .tmp2x, .memi(.src0x, .tmp0), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, - .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_2, null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, - .{ ._, .p_q, .cmpgt, .tmp5x, .dst0x, ._, ._ }, - .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, - .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._l, .cmov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .signed_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._b, .cmov, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._l, .cmov, .tmp1p, .tmp3p, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, - .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._b, .cmov, .tmp1p, .tmp3p, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, - .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, - .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp2y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp0y, .tmp0y, .tmp2y, .tmp1y }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, .v_ps, .max, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp5y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp5y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .max, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .blendv, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .@"and", .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .@"or", .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .max, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .max, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ .@"0:", .v_ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .max, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._sd, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, - .{ ._, ._sd, .max, .tmp1x, .tmp0x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp0x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .max, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .max, .tmp1x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_sd, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .tmp1x, .src0x, .tmp0x, ._ }, - .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_pd, .max, .tmp1y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_pd, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .tmp1x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_pd, .max, .tmp2y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .tmp2x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_sd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._pd, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._sd, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._pd, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._pd, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._sd, .max, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_pd, .max, .tmp4y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ .@"0:", .v_pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, - .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._pd, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._sd, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, - .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._pd, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, - .{ ._, ._sd, .max, .tmp4x, .tmp3x, ._, ._ }, - .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, - .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, - .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ ._, .f_, .xam, ._, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - .Add => comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_b, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_b, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_b, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_b, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_b, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_b, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_b, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_b, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_b, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_b, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .add, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .add, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_w, .add, .dst0x, .src0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_w, .add, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_w, .add, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_w, .add, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_w, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_w, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_w, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", ._, .add, .dst0w, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_d, .add, .dst0x, .src0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .ph_d, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_d, .add, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .ph_d, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vph_d, .add, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .ph_d, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_d, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_d, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_d, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_d, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", ._, .add, .dst0d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_q, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_q, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, - .{ ._, .vp_q, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_q, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_q, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_q, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_q, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .vp_q, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_q, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_q, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ .@"0:", .vp_q, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_q, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", .p_q, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_q, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", ._, .add, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .u64, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1p, .sa(.dst0, .sub_size_div_8), ._, ._ }, - .{ ._, ._, .lea, .tmp2q, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"1:", ._, .mov, .tmp3q, .leasia(.tmp2q, .@"8", .tmp1, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .memsia(.dst0q, .@"8", .tmp1, .add_size), .tmp3q, ._, ._ }, - .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .vh_ps, .add, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, - .{ .@"0:", .vp_w, .insr, .dst0x, .dst0x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(1) }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, - .{ .@"0:", .vp_w, .insr, .dst0x, .dst0x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(1) }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .mem(.src0w), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .mem(.src0w), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_ps, .add, .dst0x, .src0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_ps, .add, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .v_ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, - .{ .@"0:", .v_ss, .add, .dst0x, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, - .{ .@"0:", ._ss, .add, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_pd, .add, .dst0x, .src0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .h_pd, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_pd, .add, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .v_sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, - .{ .@"0:", .v_sd, .add, .dst0x, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, - .{ .@"0:", ._sd, .add, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, - .{ .@"0:", .f_, .add, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - .Mul => comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .tmp0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_q, .mov, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_b, .sub, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._q, .mov, .tmp2x, .lea(.tmp0q), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_d, .shuf, .dst0x, .tmp0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_b, .sub, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .srl, .tmp0y, .src0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .src0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .src0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp1x, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .tmp0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .tmp0x, .tmp0x, .tmp1x, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_b, .sub, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_w, .srl, .tmp2y, .dst0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_b, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_b, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .tmp2x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .vp_w, .srl, .tmp1y, .dst0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, .vp_w, .srl, .tmp1y, .tmp0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .tmp0y, .tmp0y, .tmp1y, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_b, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_b, .sub, .dst0y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_w, .srl, .tmp3y, .tmp2y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_w, .srl, .tmp3y, .dst0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ ._, .vp_w, .srl, .tmp1y, .dst0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ .@"0:", .p_w, .movzxb, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_b, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_b, .sub, .dst0y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_w, .srl, .tmp3y, .tmp2y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_w, .srl, .tmp3y, .dst0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memid(.src0x, .tmp0, 16), ._, ._ }, - .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .vp_w, .movzxb, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_b, .sub, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vp_w, .srl, .tmp2y, .dst0y, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-40, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_b, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_b, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .vp_w, .srl, .tmp3x, .tmp2x, .ui(8), ._ }, - .{ ._, .vp_w, .mull, .tmp2x, .tmp2x, .tmp3x, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_w, .movzxb, .tmp2x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", .p_w, .movzxb, .tmp2x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, - .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp2x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .al }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .mul, .memi(.src0b, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .byte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .al }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, - .{ .@"0:", ._, .mul, .memi(.src0b, .tmp0), ._, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ }, - .{ ._, .vp_w, .sub, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_w, .sub, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_w, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_w, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_w, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_w, .sub, .dst0y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_w, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_w, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_w, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_w, .sub, .dst0y, .tmp3y, .dst0y, ._ }, - .{ .@"0:", .vp_w, .mull, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_w, .sub, .dst0y, .dst0y, .tmp2y, ._ }, - .{ .@"0:", .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_w, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_w, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_w, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", .p_w, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, - .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ .@"0:", .i_, .mul, .dst0w, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .tmp0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_d, .sub, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ }, - .{ ._, .vp_d, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_d, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .vp_d, .mull, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .vp_d, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .vp_d, .sub, .dst0y, .tmp3y, .dst0y, ._ }, - .{ ._, .vp_d, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .vp_d, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .p_d, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .tmp1x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_d, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .vp_d, .sub, .dst0y, .tmp3y, .dst0y, ._ }, - .{ .@"0:", .vp_d, .mull, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .mull, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, - .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_d, .sub, .dst0y, .dst0y, .tmp2y, ._ }, - .{ .@"0:", .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .vp_d, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, - .{ ._, .vp_d, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .vp_d, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_d, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", .p_d, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .p_d, .sub, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .tmp2x, .ui(0b11_11_01_01), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, - .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .int = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, - .{ .@"0:", .i_, .mul, .dst0d, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .int = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, - .{ .@"0:", .i_, .mul, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .adx, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .isize, .kind = .{ .reg = .rcx } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"1:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, - .{ ._, ._, .@"or", .tmp4q, .leai(.tmp1q, .tmp3), ._, ._ }, - .{ ._, ._z, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp2q, .tmp5), ._ }, - .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_elem_size, -8), .tmp7q, ._, ._ }, - .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ }, - .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, - .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, - .{ ._, ._mp, .j, .@"2b", ._, ._, ._ }, - .{ .@"3:", ._, .mov, .memi(.dst0q, .tmp3), .tmp4q, ._, ._ }, - .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp3d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, - .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, - .{ .@"3:", ._x, .mul, .tmp9q, .tmp8q, .leasi(.tmp2q, .@"8", .tmp4), ._ }, - .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, - .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, - .{ .@"3:", ._x, .mul, .tmp9q, .tmp8q, .leasi(.tmp2q, .@"8", .tmp4), ._ }, - .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, - .{ .@"3:", ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp4), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._, .sub, .tmp3d, .si(1), ._, ._ }, - .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, - .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, - .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, - .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, - .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, - .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, - .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, - .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, - .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, - .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, - .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, - .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, - .{ .@"3:", ._, .mov, .tmp8q, .tmp5q, ._, ._ }, - .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp4), ._, ._, ._ }, - .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, - .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, - .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, - .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, - .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, - .{ ._, ._c, .de, .tmp3d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, - .{ .@"0:", .vp_w, .insr, .dst0x, .dst0x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(1) }, - .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .mem(.src0w), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .mem(.src0w), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .mul, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .v_ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, - .{ .@"0:", .v_ss, .mul, .dst0x, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, - .{ .@"0:", ._ss, .mul, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._sd, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .v_sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, - .{ .@"0:", .v_sd, .mul, .dst0x, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, - .{ .@"0:", ._sd, .mul, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .dst0q, ._, ._, ._ }, - .{ .@"1:", .f_, .mul, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s}.{s} {f} {f}", .{ - @tagName(air_tag), - @tagName(reduce.operation), - cg.typeOf(reduce.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - switch (reduce.operation) { - .And, .Or, .Xor, .Min, .Max => {}, - .Add, .Mul => if (cg.intInfo(res_ty)) |_| res[0].wrapInt(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s}.{s} wrap {f} {f}", .{ - @tagName(air_tag), - @tagName(reduce.operation), - res_ty.fmt(pt), - res[0].tracking(cg), - }), - else => |e| return e, - }, - } - try res[0].finish(inst, &.{reduce.operand}, &ops, cg); - }, - .reduce_optimized => |air_tag| { - const inf = std.math.inf(f16); - const reduce = air_datas[@intFromEnum(inst)].reduce; - var ops = try cg.tempsFromOperands(inst, .{reduce.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (reduce.operation) { - .And, .Or, .Xor => unreachable, - .Min => comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp0y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .min, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .min, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .min, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ss, .min, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .min, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .min, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ss, .min, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .min, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .v_ps, .min, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .min, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ss, .min, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._sd, .min, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .min, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_pd, .min, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._sd, .min, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .min, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_pd, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_pd, .min, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .v_pd, .min, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._sd, .min, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - .Max => comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp0y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .max, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .max, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .max, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ss, .max, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .max, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .max, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ss, .max, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .max, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .v_ps, .max, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .max, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ss, .max, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._sd, .max, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .max, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_pd, .max, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._sd, .max, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .max, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_pd, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_pd, .max, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .v_pd, .max, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._sd, .max, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_b, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .{ .type = .f64, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, .cmov, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_b, .cmov, .tmp2t, .tmp1t, ._, ._ }, - .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sahf, .x87, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .sahf, ._, ._, ._, ._ }, - .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, - .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, - .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - .Add => comptime &.{ .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .vh_ps, .add, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .add, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .add, .tmp0y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .tmp2y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .add, .tmp2y, .tmp2y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_ps, .add, .dst0x, .src0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_ps, .add, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_ps, .add, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .add, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .add, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .v_ps, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .v_ps, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .v_ps, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, - .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .add, .dst0x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp2x, .tmp2x, .ui(0b01), ._ }, - .{ ._, ._ss, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_pd, .add, .dst0x, .src0x, .src0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .h_pd, .add, .dst0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_pd, .add, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vh_pd, .add, .dst0y, .src0y, .src0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, - .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vh_pd, .add, .dst0y, .dst0y, .dst0y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._sd, .add, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ .@"0:", .v_pd, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .vh_pd, .add, .dst0x, .dst0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, .fast_hops, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp2x, .dst0x, ._, ._ }, - .{ ._, ._sd, .add, .dst0x, .tmp2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .add, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp1), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .add, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - .Mul => comptime &.{ .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .mul, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .mul, .tmp0y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp1y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .mul, .tmp3y, .tmp3y, .tmp4y, ._ }, - .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp4y, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .f16c, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, - .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .word }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ss, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .mul, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .mul, .dst0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_ps, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mul, .dst0x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .mul, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .v_ps, .mul, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .dword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .mul, .dst0x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, - .{ ._, ._ss, .mul, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._sd, .mul, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_sd, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .tmp0x, .dst0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, - .{ ._, .v_pd, .mul, .dst0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, - .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp0y, ._ }, - .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, - .{ .@"0:", .v_pd, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp1x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._pd, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, - .{ ._, ._sd, .mul, .dst0x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx512f, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .mul, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, - .{ ._, .v_pd, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp3y, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, - .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, - .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, - .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - .{ .@"0:", .v_pd, .mul, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, - .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, - .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", ._pd, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, - .{ ._, ._sd, .mul, .dst0x, .tmp3x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .qword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .reg = .st7 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, - .{ .@"0:", .f_, .mul, .memi(.src0q, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .gnu, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - } }, - }, .{ - .required_abi = .msvc, - .required_features = .{ .x87, null, null, null }, - .dst_constraints = .{ .{ .float = .tbyte }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .mem }, - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .reg = .st6 } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .x87 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, - .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, - .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, - .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, - .{ ._, .f_p, .mul, ._, ._, ._, ._ }, - .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, - .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .float = .xword }, .any }, - .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - } }, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s}.{s} {f} {f}", .{ - @tagName(air_tag), - @tagName(reduce.operation), - cg.typeOf(reduce.operand).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{reduce.operand}, &ops, cg); - }, - .splat => |air_tag| fallback: { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - if (cg.typeOf(ty_op.operand).toIntern() == .bool_type) break :fallback try cg.airSplat(inst); - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res: [1]Temp = undefined; - cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .broadcast, .dst0x, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .src0x, .tmp0x, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, .p_d, .shuf, .dst0x, .dst0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_b, .broadcast, .dst0y, .src0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .vp_b, .broadcast, .tmp1y, .src0b, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .tmp0x, .tmp0x, .tmp0x, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .tmp0x, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp1), .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .tmp0x, ._, ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp1), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, .p_d, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .memi(.dst0b, .tmp0), .src0b, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .memi(.dst0b, .tmp0), .src0b, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .broadcast, .dst0x, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, .p_d, .shuf, .dst0x, .dst0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .broadcast, .dst0y, .src0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .vp_w, .broadcast, .tmp1y, .src0w, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, - .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp2x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .ssse3, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_b, .shuf, .src0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_d, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .word, .is = .word } }, .any }, - .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .memi(.dst0w, .tmp0), .src0w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .broadcast, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .broadcast, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .broadcast, .dst0x, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b00_00_00_00) }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .broadcast, .dst0y, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .broadcast, .dst0y, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .vp_d, .broadcast, .tmp1y, .src0d, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp1y, .src0d, ._, ._ }, - .{ .@"0:", .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .v_ss, .broadcast, .tmp1x, .src0d, ._, ._ }, - .{ .@"0:", .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .vp_d, .shuf, .tmp1x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp1x, .src0x, .src0x, .ui(0b00_00_00_00) }, - .{ .@"0:", .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._ps, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._ps, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, - .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .dword, .is = .dword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .memi(.dst0d, .tmp0), .src0d, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .broadcast, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .movddup, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_, .movddup, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .movddup, .dst0x, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b01_00_01_00), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .shuf, .dst0x, .src0x, .ui(0b0_0), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .sse, .none, .none } }, - }, - .dst_temps = .{ .{ .rc = .sse }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b01_00_01_00), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._pd, .shuf, .dst0x, .src0x, .ui(0b0_0), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01_00_01_00), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_q, .broadcast, .dst0y, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .broadcast, .dst0y, .src0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .vp_q, .broadcast, .tmp1y, .src0q, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_sd, .broadcast, .tmp1y, .src0q, ._, ._ }, - .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .v_, .movddup, .tmp1x, .src0q, ._, ._ }, - .{ .@"0:", .v_pd, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse3, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._, .movddup, .tmp1x, .src0q, ._, ._ }, - .{ .@"0:", ._pd, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b01_00_01_00), ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._pd, .shuf, .src0x, .src0x, .ui(0b0_0), ._ }, - .{ .@"0:", ._pd, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b01_00_01_00), ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._pd, .shuf, .src0x, .src0x, .ui(0b0_0), ._ }, - .{ .@"0:", ._pd, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._ps, .shuf, .src0x, .src0x, .ui(0b01_00_01_00), ._ }, - .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._, .mov, .memi(.dst0q, .tmp0), .src0q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .broadcast, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .broadcast, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u128, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_i128, .broadcast, .tmp1y, .src0x, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u128, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_i128, .insert, .tmp1y, .src0y, .src0x, .ui(1) }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_f128, .broadcast, .tmp1y, .src0x, ._, ._ }, - .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_f128, .insert, .tmp1y, .src0y, .src0x, .ui(1) }, - .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .xword } }, .any }, - .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx2, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .broadcast, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx2, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_i128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .broadcast, .dst0y, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_f128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx2, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u128, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_i128, .broadcast, .tmp1y, .src0x, ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, - .{ ._, .v_f128, .broadcast, .tmp1y, .src0x, ._, ._ }, - .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .avx, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .mem(.src0x), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse2, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .mem(.src0x), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .sse, null, null }, - .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, - .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .mem(.src0x), ._, ._ }, - .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .slow_incdec, null, null }, - .dst_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2q, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, - .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .dst_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any }, - .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .reg = .rdi } }, - .{ .type = .usize, .kind = .{ .reg = .rsi } }, - .{ .type = .usize, .kind = .{ .reg = .rcx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_len), ._, ._ }, - .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp2q, .mem(.src0), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_size_div_8), ._, ._ }, - .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, - .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - - .memset => try cg.airMemset(inst, false), - .memset_safe => try cg.airMemset(inst, true), - .memcpy, .memmove => |air_tag| { - const bin_op = air_datas[@intFromEnum(inst)].bin_op; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }) ++ .{undefined}; - ops[2] = ops[0].getByteLen(cg) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - cg.typeOf(bin_op.rhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - }), - else => |e| return e, - }; - try ops[0].toSlicePtr(cg); - cg.select(&.{}, &.{}, &ops, switch (air_tag) { - else => unreachable, - inline .memcpy, .memmove => |symbol| comptime &.{.{ - .patterns = &.{ - .{ .src = .{ - .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .{ .to_param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } }, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = @tagName(symbol) } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }}, - }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - cg.typeOf(bin_op.rhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - ops[2].tracking(cg), - }), - else => |e| return e, - }; - for (ops) |op| try op.die(cg); - }, - .cmpxchg_weak, .cmpxchg_strong => try cg.airCmpxchg(inst), - .atomic_load => try cg.airAtomicLoad(inst), - .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered), - .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic), - .atomic_store_release => try cg.airAtomicStore(inst, .release), - .atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst), - .atomic_rmw => try cg.airAtomicRmw(inst), - .is_named_enum_value => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"32" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0p, .src0p, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"32" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0p, .src0p, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"8" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0p, .src0p, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"8" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .tag_name => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{.slice_const_u8_sentinel_0}, &ops, comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"32" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .zigcc, .after = 1, .at = 1 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"32" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p0, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .zigcc, .after = 1, .at = 1 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p0, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"8" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr = .{ .cc = .zigcc, .after = 1, .at = 1 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any_int, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"8" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .dst0p0, .mem(.src0), ._, ._ }, - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - if (res[0].tracking(cg).short != .register_pair) try ops[0].toPair(&res[0], cg); - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .error_name => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [2]Temp = undefined; - cg.select(&res, &.{ .slice_const_u8_sentinel_0, .usize }, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp1d, .src0b, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .lea(.tmp0), ._, ._ }, - .{ ._, ._, .mov, .dst1d, .leasid(.dst0d, .@"4", .tmp1, (2 - 1) * 4), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .leasid(.dst0d, .@"4", .tmp1, (1 - 1) * 4), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .not, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst1d, .leai(.dst1, .tmp1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp1d, .src0w, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .lea(.tmp0), ._, ._ }, - .{ ._, ._, .mov, .dst1d, .leasid(.dst0d, .@"4", .tmp1, (2 - 1) * 4), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .leasid(.dst0d, .@"4", .tmp1, (1 - 1) * 4), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .not, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst1d, .leai(.dst1, .tmp1), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, - .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, - .{ ._, ._, .lea, .dst0p, .lea(.tmp0), ._, ._ }, - .{ ._, ._, .mov, .dst1d, .leasid(.dst0d, .@"4", .tmp1, (2 - 1) * 4), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .leasid(.dst0d, .@"4", .tmp1, (1 - 1) * 4), ._, ._ }, - .{ ._, ._, .lea, .dst0p, .leai(.dst0, .tmp1), ._, ._ }, - .{ ._, ._, .not, .tmp1d, ._, ._, ._ }, - .{ ._, ._, .lea, .dst1d, .leai(.dst1, .tmp1), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(un_op).fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - const orig_res = res; - try res[0].toPair(&res[1], cg); - for (&ops) |*op| for (orig_res) |orig_r| { - if (op.index != orig_r.index) continue; - op.* = res[0]; - break; - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .error_set_has_value => |air_tag| { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}) ++ .{try cg.tempInit(ty_op.ty.toType(), .none)}; - var res: [1]Temp = undefined; - cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"32" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src1 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src1 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .gpr }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, - }, - .call_frame = .{ .alignment = .@"8" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src1 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .nz }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ - @tagName(air_tag), - ty_op.ty.toType().fmt(pt), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - for (ops[1..]) |op| try op.die(cg); - try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg); - }, - .aggregate_init => |air_tag| fallback: { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const agg_ty = ty_pl.ty.toType(); - if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or - (agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst); - var res = try cg.tempAllocMem(agg_ty); - const reset_index = cg.next_temp_index; - var bt = cg.liveness.iterateBigTomb(inst); - switch (ip.indexToKey(agg_ty.toIntern())) { - inline .array_type, .vector_type => |sequence_type| { - const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..@intCast(sequence_type.len)]); - const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu); - var elem_disp: u31 = 0; - for (elems) |elem_ref| { - var elem = try cg.tempFromOperand(elem_ref, bt.feed()); - try res.write(&elem, .{ .disp = elem_disp }, cg); - try elem.die(cg); - try cg.resetTemps(reset_index); - elem_disp += @intCast(elem_size); - } - if (@hasField(@TypeOf(sequence_type), "sentinel") and sequence_type.sentinel != .none) { - var sentinel = try cg.tempFromValue(.fromInterned(sequence_type.sentinel)); - try res.write(&sentinel, .{ .disp = elem_disp }, cg); - try sentinel.die(cg); - } - }, - .struct_type => { - const loaded_struct = ip.loadStructType(agg_ty.toIntern()); - const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]); - switch (loaded_struct.layout) { - .auto, .@"extern" => { - for (elems, 0..) |elem_ref, field_index| { - const elem_dies = bt.feed(); - if (loaded_struct.fieldIsComptime(ip, field_index)) continue; - if (!hack_around_sema_opv_bugs or Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]).hasRuntimeBitsIgnoreComptime(zcu)) { - var elem = try cg.tempFromOperand(elem_ref, elem_dies); - try res.write(&elem, .{ .disp = @intCast(loaded_struct.offsets.get(ip)[field_index]) }, cg); - try elem.die(cg); - try cg.resetTemps(reset_index); - } - } - }, - .@"packed" => return cg.fail("failed to select {s} {f}", .{ - @tagName(air_tag), - agg_ty.fmt(pt), - }), - } - }, - .tuple_type => |tuple_type| { - const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]); - var elem_disp: u31 = 0; - for (elems, 0..) |elem_ref, field_index| { - const elem_dies = bt.feed(); - if (tuple_type.values.get(ip)[field_index] != .none) continue; - const field_type = Type.fromInterned(tuple_type.types.get(ip)[field_index]); - elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp)); - var elem = try cg.tempFromOperand(elem_ref, elem_dies); - try res.write(&elem, .{ .disp = elem_disp }, cg); - try elem.die(cg); - try cg.resetTemps(reset_index); - elem_disp += @intCast(field_type.abiSize(zcu)); - } - }, - else => return cg.fail("failed to select {s} {f}", .{ - @tagName(air_tag), - agg_ty.fmt(pt), - }), - } - try res.finish(inst, &.{}, &.{}, cg); - }, - .union_init => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const union_init = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; - const union_ty = ty_pl.ty.toType(); - var ops = try cg.tempsFromOperands(inst, .{union_init.init}); - var res = try cg.tempAllocMem(union_ty); - const union_layout = union_ty.unionGetLayout(zcu); - if (union_layout.tag_size > 0) { - var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex( - union_ty.unionTagTypeSafety(zcu).?, - union_init.field_index, - )); - try res.write(&tag_temp, .{ - .disp = @intCast(union_layout.tagOffset()), - }, cg); - try tag_temp.die(cg); - } - try res.write(&ops[0], .{ - .disp = @intCast(union_layout.payloadOffset()), - }, cg); - try res.finish(inst, &.{union_init.init}, &ops, cg); - }, - .prefetch => { - const prefetch = air_datas[@intFromEnum(inst)].prefetch; - var ops = try cg.tempsFromOperands(inst, .{prefetch.ptr}); - switch (prefetch.cache) { - .instruction => {}, // prefetchi requires rip-relative addressing, which is currently non-trivial to emit from an arbitrary ptr value - .data => if (prefetch.rw == .write and prefetch.locality <= 2 and cg.hasFeature(.prefetchwt1)) { - try ops[0].toSlicePtr(cg); - while (try ops[0].toLea(cg)) {} - try cg.asmMemory(.{ ._wt1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); - } else if (prefetch.rw == .write and cg.hasFeature(.prfchw)) { - try ops[0].toSlicePtr(cg); - while (try ops[0].toLea(cg)) {} - try cg.asmMemory(.{ ._w, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); - } else if (cg.hasFeature(.sse) or cg.hasFeature(.prfchw) or cg.hasFeature(.prefetchi) or cg.hasFeature(.prefetchwt1)) { - try ops[0].toSlicePtr(cg); - while (try ops[0].toLea(cg)) {} - switch (prefetch.locality) { - 0 => try cg.asmMemory(.{ ._nta, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), - 1 => try cg.asmMemory(.{ ._t2, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), - 2 => try cg.asmMemory(.{ ._t1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), - 3 => try cg.asmMemory(.{ ._t0, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), - } - }, - } - const res = try cg.tempInit(.void, .none); - try res.finish(inst, &.{prefetch.ptr}, &ops, cg); - }, - .mul_add => |air_tag| { - const pl_op = air_datas[@intFromEnum(inst)].pl_op; - const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs, pl_op.operand }); - var res: [1]Temp = undefined; - cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ - .required_features = .{ .fma, .f16c, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .to_sse, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .src2q, ._, ._ }, - .{ ._, .v_ss, .fmadd213, .dst0x, .tmp0x, .tmp1d, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - .{ .scalar_float = .{ .of = .word, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fma, .f16c, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_float = .{ .of = .qword, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .mem } }, - .{ .src = .{ .to_sse, .mem, .mem } }, - .{ .src = .{ .mem, .to_sse, .mem } }, - .{ .src = .{ .mem, .mem, .to_sse } }, - .{ .src = .{ .to_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_sse, .to_sse } }, - .{ .src = .{ .to_sse, .to_sse, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .vector_4_f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_4_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1x, .src2q, ._, ._ }, - .{ ._, .v_ps, .fmadd213, .dst0x, .tmp0x, .tmp1x, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .fma, .f16c, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_float = .{ .of = .xword, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .mem } }, - .{ .src = .{ .to_sse, .mem, .mem } }, - .{ .src = .{ .mem, .to_sse, .mem } }, - .{ .src = .{ .mem, .mem, .to_sse } }, - .{ .src = .{ .to_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_sse, .to_sse } }, - .{ .src = .{ .to_sse, .to_sse, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp1y, .src2x, ._, ._ }, - .{ ._, .v_ps, .fmadd213, .dst0y, .tmp0y, .tmp1y, ._ }, - .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, - } }, - }, .{ - .required_features = .{ .fma, .f16c, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp2y, .memi(.src1x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .cvtph2, .tmp3y, .memi(.src2x, .tmp0), ._, ._ }, - .{ ._, .v_ps, .fmadd213, .tmp1y, .tmp2y, .tmp3y, ._ }, - .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, - .{ ._, .vp_w, .insr, .tmp1x, .tmp3x, .memi(.src0w, .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memi(.src1w, .tmp0), .ui(0) }, - .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memi(.src2w, .tmp0), .ui(0) }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memi(.src1w, .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp3x, .memi(.src2w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp2x, .memi(.src1w, .tmp0), .ui(0), ._ }, - .{ ._, .p_w, .insr, .tmp3x, .memi(.src2w, .tmp0), .ui(0), ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp5w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f16, .kind = .{ .reg = .ax } }, - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memi(.src1w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .movzx, .tmp1d, .memi(.src2w, .tmp0), ._, ._ }, - .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, - .{ ._, ._ss, .mov, .tmp5x, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .fmadd132, .dst0x, .src2x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .fmadd213, .dst0x, .src1x, .src2d, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, - .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, - }, - .dst_temps = .{ .{ .ref = .src2 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .fmadd231, .dst0x, .src0x, .src1d, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .scalar_float = .{ .of = .dword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmaf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .fmadd132, .dst0x, .src2x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .fmadd213, .dst0x, .src1x, .src2x, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - .{ .scalar_float = .{ .of = .xword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, - .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, - }, - .dst_temps = .{ .{ .ref = .src2 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .fmadd231, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .fmadd132, .dst0y, .src2y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .fmadd213, .dst0y, .src1y, .src2y, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .scalar_float = .{ .of = .yword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, - .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, - }, - .dst_temps = .{ .{ .ref = .src2 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_ps, .fmadd231, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_ps, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .mova, .tmp2y, .memi(.src1y, .tmp0), ._, ._ }, - .{ ._, .v_ps, .fmadd213, .tmp1y, .tmp2y, .memi(.src2y, .tmp0), ._ }, - .{ ._, .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, .v_ss, .mov, .tmp2x, .memi(.src1d, .tmp0), ._, ._ }, - .{ ._, .v_ss, .mov, .tmp3x, .memi(.src2d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaf" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, - .{ ._, ._ss, .mov, .tmp2x, .memi(.src1d, .tmp0), ._, ._ }, - .{ ._, ._ss, .mov, .tmp3x, .memi(.src2d, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .fmadd132, .dst0x, .src2x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .fmadd213, .dst0x, .src1x, .src2q, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, - .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, - }, - .dst_temps = .{ .{ .ref = .src2 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .fmadd231, .dst0x, .src0x, .src1q, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .fmadd132, .dst0x, .src2x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .fmadd213, .dst0x, .src1x, .src2x, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - .{ .scalar_float = .{ .of = .xword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, - .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, - }, - .dst_temps = .{ .{ .ref = .src2 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .fmadd231, .dst0x, .src0x, .src1x, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, - .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .fmadd132, .dst0y, .src2y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, - .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mut_sse, .sse, .sse } }, - .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .fmadd213, .dst0y, .src1y, .src2y, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .scalar_float = .{ .of = .yword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, - .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, - }, - .dst_temps = .{ .{ .ref = .src2 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .fmadd231, .dst0y, .src0y, .src1y, ._ }, - } }, - }, .{ - .required_features = .{ .fma, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, - .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, - .{ .@"0:", .v_pd, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, - .{ ._, .v_pd, .mova, .tmp2y, .memi(.src1y, .tmp0), ._, ._ }, - .{ ._, .v_pd, .fmadd213, .tmp1y, .tmp2y, .memi(.src2y, .tmp0), ._ }, - .{ ._, .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, .v_sd, .mov, .tmp2x, .memi(.src1q, .tmp0), ._, ._ }, - .{ ._, .v_sd, .mov, .tmp3x, .memi(.src2q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._sd, .mov, .tmp2x, .memi(.src1q, .tmp0), ._, ._ }, - .{ ._, ._sd, .mov, .tmp3x, .memi(.src2q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, - .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, - .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._ps, .movl, .tmp2x, .memi(.src1q, .tmp0), ._, ._ }, - .{ ._, ._ps, .movl, .tmp3x, .memi(.src2q, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .size = 16 * 3, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .reg = .st0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp0x, .mem(.src2x), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp1x, 16 * 2), .tmp0x, ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .mem(.src2), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .required_features = .{ .sse, .x87, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .size = 16 * 3, .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f80, .kind = .{ .frame = .call_frame } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .memi(.src1x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .memi(.src2x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .memd(.tmp2x, 16 * 2), .tmp1x, ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, - .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memi(.src1, .tmp0), ._, ._ }, - .{ ._, ._, .lea, .tmp4p, .memi(.src2, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .scalar_float = .{ .of = .xword, .is = .xword } }, - }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, - .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp3x, .memi(.src2x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp3x, .memi(.src2x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .to_mem } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ }, - .{ ._, ._ps, .mova, .tmp3x, .memi(.src2x, .tmp0), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, - .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ - @tagName(air_tag), - cg.typeOf(bin_op.lhs).fmt(pt), - ops[0].tracking(cg), - ops[1].tracking(cg), - ops[2].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand }, &ops, cg); - }, - .field_parent_ptr => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const field_parent_ptr = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; - var ops = try cg.tempsFromOperands(inst, .{field_parent_ptr.field_ptr}); - try ops[0].toOffset(-@as(i32, @intCast(codegen.fieldOffset( - ty_pl.ty.toType(), - cg.typeOf(field_parent_ptr.field_ptr), - field_parent_ptr.field_index, - zcu, - ))), cg); - try ops[0].finish(inst, &.{field_parent_ptr.field_ptr}, &ops, cg); - }, - .wasm_memory_size, .wasm_memory_grow => unreachable, - .cmp_lt_errors_len => |air_tag| { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - var res: [1]Temp = undefined; - cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .be }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .lea(.tmp0), ._, ._ }, - .{ ._, ._, .cmp, .src0b, .lea(.tmp1b), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .be }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .lea(.tmp0), ._, ._ }, - .{ ._, ._, .cmp, .src0w, .lea(.tmp1w), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .be }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp1p, .lea(.tmp0), ._, ._ }, - .{ ._, ._, .cmp, .src0d, .lea(.tmp1d), ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {f}", .{ - @tagName(air_tag), - ops[0].tracking(cg), - }), - else => |e| return e, - }; - try res[0].finish(inst, &.{un_op}, &ops, cg); - }, - .err_return_trace => { - const ert: Temp = .{ .index = err_ret_trace_index }; - try ert.finish(inst, &.{}, &.{}, cg); - }, - .set_err_return_trace => { - const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - switch (ops[0].unwrap(cg)) { - .ref => { - const result = try cg.allocRegOrMem(err_ret_trace_index, true); - try cg.genCopy(.usize, result, ops[0].tracking(cg).short, .{}); - tracking_log.debug("{f} => {f} (birth)", .{ err_ret_trace_index, result }); - cg.inst_tracking.putAssumeCapacityNoClobber(err_ret_trace_index, .init(result)); - }, - .temp => |temp_index| { - const temp_tracking = temp_index.tracking(cg); - tracking_log.debug("{f} => {f} (birth)", .{ err_ret_trace_index, temp_tracking.short }); - cg.inst_tracking.putAssumeCapacityNoClobber(err_ret_trace_index, temp_tracking.*); - assert(cg.reuseTemp(err_ret_trace_index, temp_index.toIndex(), temp_tracking)); - }, - .err_ret_trace => unreachable, - } - }, - .addrspace_cast => { - const ty_op = air_datas[@intFromEnum(inst)].ty_op; - const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); - }, - .save_err_return_trace_index => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const agg_ty = ty_pl.ty.toType(); - assert(agg_ty.containerLayout(zcu) != .@"packed"); - var ert: Temp = .{ .index = err_ret_trace_index }; - var res = try ert.load(.usize, .{ .disp = @intCast(agg_ty.structFieldOffset(ty_pl.payload, zcu)) }, cg); - try ert.die(cg); - try res.finish(inst, &.{}, &.{}, cg); - }, - .vector_store_elem => { - const vector_store_elem = air_datas[@intFromEnum(inst)].vector_store_elem; - const bin_op = cg.air.extraData(Air.Bin, vector_store_elem.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ vector_store_elem.vector_ptr, bin_op.lhs, bin_op.rhs }); - cg.select(&.{}, &.{}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._s, .bt, .lea(.src0w), .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .dword }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .lea(.src0d), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0d), .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .qword }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .lea(.src0q), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._r, .bt, .tmp1q, .src1q, ._, ._ }, - .{ ._, ._s, .bt, .tmp0q, .src1q, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0q), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(5), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .leasi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .bt, .tmp2d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .leasi(.src0d, .@"4", .tmp0), .tmp1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .byte } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .imm8 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0b, .add_src0_elem_size_mul_src1), .src2b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .byte } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .imm8 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leai(.src0b, .src1), .src2b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .imm16 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .imm16 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, - .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, - .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, - .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, - .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .imm32 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .imm32 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any, .any, .{ .int = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .simm32 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any, .any, .{ .int = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .simm32 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => { - const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu); - while (try ops[0].toRegClass(true, .general_purpose, cg) or - try ops[1].toRegClass(true, .general_purpose, cg)) - {} - const base_reg = ops[0].tracking(cg).short.register.to64(); - const rhs_reg = ops[1].tracking(cg).short.register.to64(); - if (!std.math.isPowerOfTwo(elem_size)) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - rhs_reg, - rhs_reg, - .u(elem_size), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ - .base = .{ .reg = base_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else if (elem_size > 8) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterImmediate( - .{ ._l, .sh }, - rhs_reg, - .u(std.math.log2_int(u64, elem_size)), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ - .base = .{ .reg = base_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ - .base = .{ .reg = base_reg }, - .mod = .{ .rm = .{ - .index = rhs_reg, - .scale = .fromFactor(@intCast(elem_size)), - } }, - }); - try ops[0].store(&ops[2], .{}, cg); - }, - else => |e| return e, - }; - for (ops) |op| try op.die(cg); - }, - .runtime_nav_ptr => { - const ty_nav = air_datas[@intFromEnum(inst)].ty_nav; - const nav = ip.getNav(ty_nav.nav); - const is_threadlocal = zcu.comp.config.any_non_single_threaded and nav.isThreadlocal(ip); - if (is_threadlocal) if (cg.mod.pic) { - try cg.spillRegisters(&.{ .rdi, .rax }); - } else { - try cg.spillRegisters(&.{.rax}); - }; - var res = try cg.tempInit(.fromInterned(ty_nav.ty), .{ .lea_nav = ty_nav.nav }); - if (is_threadlocal) while (try res.toRegClass(true, .general_purpose, cg)) {}; - try res.finish(inst, &.{}, &.{}, cg); - }, - .c_va_arg => try cg.airVaArg(inst), - .c_va_copy => try cg.airVaCopy(inst), - .c_va_end => try cg.airVaEnd(inst), - .c_va_start => try cg.airVaStart(inst), - .work_item_id, .work_group_size, .work_group_id => unreachable, - } - try cg.resetTemps(@enumFromInt(0)); - cg.checkInvariantsAfterAirInst(); - } - verbose_tracking_log.debug("{f}", .{cg.fmtTracking()}); -} - -fn genLazy(cg: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - switch (ip.indexToKey(lazy_sym.ty)) { - .enum_type => { - const enum_ty: Type = .fromInterned(lazy_sym.ty); - wip_mir_log.debug("{f}.@tagName:", .{enum_ty.fmt(pt)}); - - const ret_regs = abi.getCAbiIntReturnRegs(.auto)[0..2].*; - const ret_locks = cg.register_manager.lockRegsAssumeUnused(2, ret_regs); - defer for (ret_locks) |lock| cg.register_manager.unlockReg(lock); - - const param_reg = abi.getCAbiIntParamRegs(.auto)[0]; - const param_lock = cg.register_manager.lockReg(param_reg); - defer if (param_lock) |lock| cg.register_manager.unlockReg(lock); - - const ret_mcv: MCValue = .{ .register_pair = ret_regs }; - - const data_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); - const data_lock = cg.register_manager.lockRegAssumeUnused(data_reg); - defer cg.register_manager.unlockReg(data_lock); - try cg.asmRegisterMemory(.{ ._, .lea }, data_reg.to64(), .{ - .base = .{ .lazy_sym = .{ .kind = .const_data, .ty = lazy_sym.ty } }, - }); - - var data_off: i32 = 0; - const reset_index = cg.next_temp_index; - const tag_names = ip.loadEnumType(lazy_sym.ty).names; - for (0..tag_names.len) |tag_index| { - var enum_temp = try cg.tempInit(enum_ty, if (enum_ty.abiSize(zcu) <= @as(u4, switch (cg.target.cpu.arch) { - else => unreachable, - .x86 => 4, - .x86_64 => 8, - })) .{ .register = param_reg } else .{ .indirect = .{ .reg = param_reg } }); - - const tag_name_len = tag_names.get(ip)[tag_index].length(ip); - var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index))); - const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, cg) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try enum_temp.die(cg); - try tag_temp.die(cg); - const skip_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); - try cc_temp.die(cg); - - try cg.genSetReg( - ret_mcv.register_pair[0], - .usize, - .{ .register_offset = .{ .reg = data_reg, .off = data_off } }, - .{}, - ); - try cg.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{}); - try cg.asmOpOnly(.{ ._, .ret }); - - cg.performReloc(skip_reloc); - - data_off += @intCast(tag_name_len + 1); - - try cg.resetTemps(reset_index); - } - - try cg.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{}); - try cg.asmOpOnly(.{ ._, .ret }); - }, - .error_set_type => |error_set_type| { - const err_ty: Type = .fromInterned(lazy_sym.ty); - wip_mir_log.debug("{f}.@errorCast:", .{err_ty.fmt(pt)}); - - const ret_reg = abi.getCAbiIntReturnRegs(.auto)[0]; - const ret_lock = cg.register_manager.lockRegAssumeUnused(ret_reg); - defer cg.register_manager.unlockReg(ret_lock); - - const param_reg = abi.getCAbiIntParamRegs(.auto)[0]; - const param_lock = cg.register_manager.lockReg(param_reg); - defer if (param_lock) |lock| cg.register_manager.unlockReg(lock); - - const ret_mcv: MCValue = .{ .register = ret_reg }; - const err_mcv: MCValue = .{ .register = param_reg }; - var err_temp = try cg.tempInit(err_ty, err_mcv); - - const ExpectedContents = [32]Mir.Inst.Index; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); - const allocator = stack.get(); - - const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len); - defer allocator.free(relocs); - - const reset_index = cg.next_temp_index; - for (0.., relocs) |tag_index, *reloc| { - var tag_temp = try cg.tempInit(.anyerror, .{ - .immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?, - }); - const cc_temp = err_temp.cmpInts(.eq, &tag_temp, cg) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try tag_temp.die(cg); - reloc.* = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); - try cc_temp.die(cg); - try cg.resetTemps(reset_index); - } - try err_temp.die(cg); - - try cg.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{}); - for (relocs) |reloc| cg.performReloc(reloc); - assert(ret_mcv.register == err_mcv.register); - try cg.asmOpOnly(.{ ._, .ret }); - }, - else => return cg.fail( - "TODO implement {s} for {f}", - .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, - ), - } - try cg.resetTemps(@enumFromInt(0)); - cg.checkInvariantsAfterAirInst(); -} - -fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { - for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst); - switch (value) { - else => {}, - .eflags, .register_overflow => self.eflags_inst = inst, - } -} - -fn getValueIfFree(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) void { - for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg)) - self.register_manager.getRegAssumeFree(reg, inst); -} - -const FreeOptions = struct { emit_instructions: bool = true }; - -fn freeReg(self: *CodeGen, reg: Register, comptime opts: FreeOptions) !void { - self.register_manager.freeReg(reg); - if (opts.emit_instructions and reg.isClass(.x87)) try self.asmRegister(.{ .f_, .free }, reg); -} - -fn freeValue(self: *CodeGen, value: MCValue, comptime opts: FreeOptions) !void { - switch (value) { - .register => |reg| try self.freeReg(reg, opts), - inline .register_pair, - .register_triple, - .register_quadruple, - => |regs| for (regs) |reg| try self.freeReg(reg, opts), - .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg, opts), - .register_overflow => |reg_ov| { - try self.freeReg(reg_ov.reg, opts); - self.eflags_inst = null; - }, - .register_mask => |reg_mask| try self.freeReg(reg_mask.reg, opts), - .eflags => self.eflags_inst = null, - else => {}, // TODO process stack allocation death - } -} - -fn feed(self: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void { - if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst, .{}); -} - -/// Asserts there is already capacity to insert into top branch inst_table. -fn processDeath(self: *CodeGen, inst: Air.Inst.Index, comptime opts: FreeOptions) !void { - try self.inst_tracking.getPtr(inst).?.die(self, inst, opts); -} - -fn finishAirResult(self: *CodeGen, inst: Air.Inst.Index, result: MCValue) void { - if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) { - .none, .dead, .unreach => {}, - else => unreachable, // Why didn't the result die? - } else { - tracking_log.debug("{f} => {f} (birth)", .{ inst, result }); - self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result)); - // In some cases, an operand may be reused as the result. - // If that operand died and was a register, it was freed by - // processDeath, so we have to "re-allocate" the register. - self.getValueIfFree(result, inst); - } -} - -fn finishAir( - self: *CodeGen, - inst: Air.Inst.Index, - result: MCValue, - operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, -) !void { - const tomb_bits = self.liveness.getTombBits(inst); - for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; - if (self.reused_operands.isSet(op_index)) continue; - try self.processDeath(op.toIndexAllowNone() orelse continue, .{}); - } - self.finishAirResult(inst, result); -} - -const FrameLayout = struct { - stack_mask: u32, - stack_adjust: u32, - save_reg_list: Mir.RegisterList, -}; - -fn setFrameLoc( - self: *CodeGen, - frame_index: FrameIndex, - base: Register, - offset: *i32, - comptime aligned: bool, -) void { - const frame_i = @intFromEnum(frame_index); - if (aligned) { - const alignment = self.frame_allocs.items(.abi_align)[frame_i]; - offset.* = @intCast(alignment.forward(@intCast(offset.*))); - } - self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* }); - offset.* += self.frame_allocs.items(.abi_size)[frame_i]; -} - -fn computeFrameLayout(self: *CodeGen, cc: std.builtin.CallingConvention.Tag) !FrameLayout { - const frame_allocs_len = self.frame_allocs.len; - try self.frame_locs.resize(self.gpa, frame_allocs_len); - const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count); - defer self.gpa.free(stack_frame_order); - - const frame_size = self.frame_allocs.items(.abi_size); - const frame_align = self.frame_allocs.items(.abi_align); - const frame_offset = self.frame_locs.items(.disp); - - for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index| - frame_order.* = @enumFromInt(frame_index); - { - const SortContext = struct { - frame_align: @TypeOf(frame_align), - pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool { - return context.frame_align[@intFromEnum(lhs)].compare(.gt, context.frame_align[@intFromEnum(rhs)]); - } - }; - const sort_context = SortContext{ .frame_align = frame_align }; - std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan); - } - - const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)]; - const stack_frame_align = frame_align[@intFromEnum(FrameIndex.stack_frame)]; - const args_frame_align = frame_align[@intFromEnum(FrameIndex.args_frame)]; - const needed_align = call_frame_align.max(stack_frame_align); - const need_align_stack = needed_align.compare(.gt, args_frame_align); - - // Create list of registers to save in the prologue. - // TODO handle register classes - var save_reg_list: Mir.RegisterList = .empty; - const callee_preserved_regs = abi.getCalleePreservedRegs(cc); - for (callee_preserved_regs) |reg| { - if (self.register_manager.isRegAllocated(reg)) { - save_reg_list.push(callee_preserved_regs, reg); - } - } - - var rbp_offset: i32 = 0; - self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false); - self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false); - self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false); - const stack_frame_align_offset = if (need_align_stack) - 0 - else - save_reg_list.size(self.target) + frame_offset[@intFromEnum(FrameIndex.args_frame)]; - - var rsp_offset: i32 = 0; - self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true); - self.setFrameLoc(.stack_frame, .rsp, &rsp_offset, true); - for (stack_frame_order) |frame_index| self.setFrameLoc(frame_index, .rsp, &rsp_offset, true); - rsp_offset += stack_frame_align_offset; - rsp_offset = @intCast(needed_align.forward(@intCast(rsp_offset))); - rsp_offset -= stack_frame_align_offset; - frame_size[@intFromEnum(FrameIndex.call_frame)] = - @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]); - - return .{ - .stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0), - .stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]), - .save_reg_list = save_reg_list, - }; -} - -fn getFrameAddrAlignment(self: *CodeGen, frame_addr: bits.FrameAddr) InternPool.Alignment { - const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align; - return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off))); -} - -fn getFrameAddrSize(self: *CodeGen, frame_addr: bits.FrameAddr) u32 { - return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off)); -} - -fn allocFrameIndex(self: *CodeGen, alloc: FrameAlloc) !FrameIndex { - const frame_allocs_slice = self.frame_allocs.slice(); - const frame_size = frame_allocs_slice.items(.abi_size); - const frame_align = frame_allocs_slice.items(.abi_align); - - const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)]; - stack_frame_align.* = stack_frame_align.max(alloc.abi_align); - - for (self.free_frame_indices.keys(), 0..) |frame_index, free_i| { - const abi_size = frame_size[@intFromEnum(frame_index)]; - if (abi_size != alloc.abi_size) continue; - const abi_align = &frame_align[@intFromEnum(frame_index)]; - abi_align.* = abi_align.max(alloc.abi_align); - - _ = self.free_frame_indices.swapRemoveAt(free_i); - return frame_index; - } - const frame_index: FrameIndex = @enumFromInt(self.frame_allocs.len); - try self.frame_allocs.append(self.gpa, alloc); - return frame_index; -} - -/// Use a pointer instruction as the basis for allocating stack memory. -fn allocMemPtr(self: *CodeGen, inst: Air.Inst.Index) !FrameIndex { - const pt = self.pt; - const zcu = pt.zcu; - const ptr_ty = self.typeOfIndex(inst); - const val_ty = ptr_ty.childType(zcu); - return self.allocFrameIndex(.init(.{ - .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse { - return self.fail("type '{f}' too big to fit into stack frame", .{val_ty.fmt(pt)}); - }, - .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"), - })); -} - -fn allocRegOrMem(self: *CodeGen, inst: Air.Inst.Index, reg_ok: bool) !MCValue { - return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok); -} - -fn allocTempRegOrMem(self: *CodeGen, elem_ty: Type, reg_ok: bool) !MCValue { - return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok); -} - -fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse { - return self.fail("type '{f}' too big to fit into stack frame", .{ty.fmt(pt)}); - }; - - if (reg_ok) need_mem: { - if (std.math.isPowerOfTwo(abi_size) and abi_size <= @as(u32, max_abi_size: switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(self.target)) { - 16, 32, 64, 128 => 16, - 80 => break :need_mem, - else => unreachable, - }, - .vector => { - const elem_ty = ty.childType(zcu); - break :max_abi_size if (elem_ty.toIntern() == .bool_type) - 8 - else if (self.floatBits(elem_ty)) |float_bits| switch (float_bits) { - 16, 32, 64, 128 => self.vectorSize(.float), - 80 => break :need_mem, - else => unreachable, - } else self.vectorSize(.int); - }, - else => 8, - })) { - if (self.register_manager.tryAllocReg(inst, self.regSetForType(ty))) |reg| { - return MCValue{ .register = registerAlias(reg, abi_size) }; - } - } - } - - const frame_index = try self.allocFrameIndex(.initSpill(ty, zcu)); - return .{ .load_frame = .{ .index = frame_index } }; -} - -fn regClassForType(self: *CodeGen, ty: Type) Register.Class { - const pt = self.pt; - const zcu = pt.zcu; - if (self.floatBits(ty)) |float_bits| return switch (float_bits) { - 80 => .x87, - else => .sse, - }; - if (!ty.isVector(zcu)) return .general_purpose; - const elem_ty = ty.childType(zcu); - return if (elem_ty.toIntern() == .bool_type) - .general_purpose - else if (self.floatBits(elem_ty)) |float_bits| - if (float_bits == 80) .x87 else .sse - else if (self.intInfo(elem_ty)) |_| - .sse - else - .general_purpose; -} - -fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet { - return switch (rc) { - .general_purpose => abi.RegisterClass.gp, - .gphi => abi.RegisterClass.gphi, - .segment, .ip, .cr, .dr => unreachable, - .x87 => abi.RegisterClass.x87, - .mmx => @panic("TODO"), - .sse => abi.RegisterClass.sse, - }; -} - -fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet { - return regSetForRegClass(self.regClassForType(ty)); -} - -fn vectorSize(cg: *CodeGen, kind: enum { int, float }) u6 { - return if (cg.hasFeature(switch (kind) { - .int => .avx2, - .float => .avx, - })) 32 else if (cg.hasFeature(.sse)) 16 else 8; -} - -const State = struct { - registers: RegisterManager.TrackedRegisters, - reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, - free_registers: RegisterManager.RegisterBitSet, - next_temp_index: Temp.Index, - inst_tracking_len: u32, - scope_generation: u32, -}; - -fn initRetroactiveState(self: *CodeGen) State { - const scope_generation = self.scope_generation + 1; - self.scope_generation = scope_generation; - - var state: State = undefined; - state.next_temp_index = @enumFromInt(0); - state.inst_tracking_len = @intCast(self.inst_tracking.count()); - state.scope_generation = scope_generation; - return state; -} - -fn saveRetroactiveState(self: *CodeGen, state: *State) !void { - try self.spillEflagsIfOccupied(); - const free_registers = self.register_manager.free_registers; - var it = free_registers.iterator(.{ .kind = .unset }); - while (it.next()) |index| { - const tracked_inst = self.register_manager.registers[index]; - state.registers[index] = tracked_inst; - state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?; - } - state.free_registers = free_registers; -} - -fn saveState(self: *CodeGen) !State { - var state = self.initRetroactiveState(); - try self.saveRetroactiveState(&state); - return state; -} - -fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct { - emit_instructions: bool, - update_tracking: bool, - resurrect: bool, - close_scope: bool, -}) !void { - if (opts.close_scope) { - for ( - self.inst_tracking.keys()[@intFromEnum(state.next_temp_index)..@intFromEnum(self.next_temp_index)], - self.inst_tracking.values()[@intFromEnum(state.next_temp_index)..@intFromEnum(self.next_temp_index)], - ) |inst, *tracking| try tracking.die(self, inst, .{ .emit_instructions = opts.emit_instructions }); - self.next_temp_index = state.next_temp_index; - for ( - self.inst_tracking.keys()[state.inst_tracking_len..], - self.inst_tracking.values()[state.inst_tracking_len..], - ) |inst, *tracking| try tracking.die(self, inst, .{ .emit_instructions = opts.emit_instructions }); - self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len); - } - - if (opts.resurrect) { - for ( - self.inst_tracking.keys()[0..@intFromEnum(state.next_temp_index)], - self.inst_tracking.values()[0..@intFromEnum(state.next_temp_index)], - ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation); - for ( - self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len], - self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len], - ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation); - } - for (deaths) |death| try self.processDeath(death, .{ .emit_instructions = opts.emit_instructions }); - - const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - if (opts.update_tracking) {} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - - var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity( - stack.get(), - @typeInfo(ExpectedContents).array.len, - ); - defer if (!opts.update_tracking) { - for (reg_locks.items) |lock| self.register_manager.unlockReg(lock); - reg_locks.deinit(); - }; - - for ( - 0.., - self.register_manager.registers, - state.registers, - state.reg_tracking, - ) |reg_i, current_slot, target_slot, reg_tracking| { - const reg_index: RegisterManager.TrackedIndex = @intCast(reg_i); - const current_maybe_inst = if (self.register_manager.isRegIndexFree(reg_index)) null else current_slot; - const target_maybe_inst = if (state.free_registers.isSet(reg_index)) null else target_slot; - if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst| - assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len); - if (opts.emit_instructions and current_maybe_inst != target_maybe_inst) { - if (current_maybe_inst) |current_inst| - try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst); - if (target_maybe_inst) |target_inst| - try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg_tracking); - } - if (opts.update_tracking) { - if (current_maybe_inst) |current_inst| { - try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst); - self.register_manager.freeRegIndex(reg_index); - } - if (target_maybe_inst) |target_inst| { - self.register_manager.getRegIndexAssumeFree(reg_index, target_inst); - self.inst_tracking.getPtr(target_inst).?.trackMaterialize(target_inst, reg_tracking); - } - } else if (target_maybe_inst) |_| - try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(reg_index)); - } - if (opts.emit_instructions) if (self.eflags_inst) |inst| - try self.inst_tracking.getPtr(inst).?.spill(self, inst); - if (opts.update_tracking) if (self.eflags_inst) |inst| { - self.eflags_inst = null; - try self.inst_tracking.getPtr(inst).?.trackSpill(self, inst); - }; - - if (opts.update_tracking and std.debug.runtime_safety) { - assert(self.eflags_inst == null); - assert(self.register_manager.free_registers.eql(state.free_registers)); - var used_reg_it = state.free_registers.iterator(.{ .kind = .unset }); - while (used_reg_it.next()) |index| - assert(self.register_manager.registers[index] == state.registers[index]); - } -} - -pub fn spillInstruction(self: *CodeGen, reg: Register, inst: Air.Inst.Index) !void { - const tracking = self.inst_tracking.getPtr(inst) orelse return; - for (tracking.getRegs()) |tracked_reg| { - if (tracked_reg.id() == reg.id()) break; - } else unreachable; // spilled reg not tracked with spilled instruction - try tracking.spill(self, inst); - try tracking.trackSpill(self, inst); -} - -pub fn spillEflagsIfOccupied(self: *CodeGen) !void { - if (self.eflags_inst) |inst| { - self.eflags_inst = null; - const tracking = self.inst_tracking.getPtr(inst).?; - assert(tracking.getCondition() != null); - try tracking.spill(self, inst); - try tracking.trackSpill(self, inst); - } -} - -pub fn spillCallerPreservedRegs(self: *CodeGen, cc: std.builtin.CallingConvention.Tag, ignore_reg: Register) !void { - switch (cc) { - inline .auto, .x86_64_sysv, .x86_64_win => |tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| - if (reg != ignore_reg) try self.register_manager.getKnownReg(reg, null), - else => unreachable, - } -} - -pub fn spillRegisters(self: *CodeGen, comptime registers: []const Register) !void { - inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null); -} - -/// Copies a value to a register without tracking the register. The register is not considered -/// allocated. A second call to `copyToTmpRegister` may return the same register. -/// This can have a side effect of spilling instructions to the stack to free up a register. -fn copyToTmpRegister(self: *CodeGen, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null, self.regSetForType(ty)); - try self.genSetReg(reg, ty, mcv, .{}); - return reg; -} - -/// Allocates a new register and copies `mcv` into it. -/// `reg_owner` is the instruction that gets associated with the register in the register table. -/// This can have a side effect of spilling instructions to the stack to free up a register. -/// WARNING make sure that the allocated register matches the returned MCValue from an instruction! -fn copyToRegisterWithInstTracking( - self: *CodeGen, - reg_owner: Air.Inst.Index, - ty: Type, - mcv: MCValue, -) !MCValue { - const reg: Register = try self.register_manager.allocReg(reg_owner, self.regSetForType(ty)); - try self.genSetReg(reg, ty, mcv, .{}); - return MCValue{ .register = reg }; -} - -fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void { - const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } }; - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const result: MCValue = switch (self.ret_mcv.long) { - else => unreachable, - .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } }, - .load_frame => .{ .register_offset = .{ - .reg = (try self.copyToRegisterWithInstTracking( - inst, - self.typeOfIndex(inst), - self.ret_mcv.long, - )).register, - .off = self.ret_mcv.short.indirect.off, - } }, - }; - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const dst_ty = self.typeOfIndex(inst); - const dst_bits = dst_ty.floatBits(self.target); - const src_ty = self.typeOf(ty_op.operand); - const src_bits = src_ty.floatBits(self.target); - - const result = result: { - if (switch (dst_bits) { - 16 => switch (src_bits) { - 32 => !self.hasFeature(.f16c), - 64, 80, 128 => true, - else => unreachable, - }, - 32 => switch (src_bits) { - 64 => false, - 80, 128 => true, - else => unreachable, - }, - 64 => switch (src_bits) { - 80, 128 => true, - else => unreachable, - }, - 80 => switch (src_bits) { - 128 => true, - else => unreachable, - }, - else => unreachable, - }) { - var sym_buf: ["__trunc?f?f2".len]u8 = undefined; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(), - .param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "__trunc{c}f{c}f2", .{ - floatCompilerRtAbiName(src_bits), - floatCompilerRtAbiName(dst_bits), - }) catch unreachable, - } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); - } - - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); - const dst_reg = dst_mcv.getReg().?.to128(); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - if (dst_bits == 16) { - assert(self.hasFeature(.f16c)); - switch (src_bits) { - 32 => { - const mat_src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - mat_src_reg.to128(), - bits.RoundMode.imm(.{}), - ); - }, - else => unreachable, - } - } else { - assert(src_bits == 64 and dst_bits == 32); - if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( - .{ .v_ss, .cvtsd2 }, - dst_reg, - dst_reg, - try src_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegisterRegister( - .{ .v_ss, .cvtsd2 }, - dst_reg, - dst_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv)).to128(), - ) else if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ ._ss, .cvtsd2 }, - dst_reg, - try src_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._ss, .cvtsd2 }, - dst_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv)).to128(), - ); - } - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const dst_ty = self.typeOfIndex(inst); - const dst_scalar_ty = dst_ty.scalarType(zcu); - const dst_bits = dst_scalar_ty.floatBits(self.target); - const src_ty = self.typeOf(ty_op.operand); - const src_scalar_ty = src_ty.scalarType(zcu); - const src_bits = src_scalar_ty.floatBits(self.target); - - const result = result: { - if (switch (src_bits) { - 16 => switch (dst_bits) { - 32, 64 => !self.hasFeature(.f16c), - 80, 128 => true, - else => unreachable, - }, - 32 => switch (dst_bits) { - 64 => false, - 80, 128 => true, - else => unreachable, - }, - 64 => switch (dst_bits) { - 80, 128 => true, - else => unreachable, - }, - 80 => switch (dst_bits) { - 128 => true, - else => unreachable, - }, - else => unreachable, - }) { - if (dst_ty.isVector(zcu)) break :result null; - var sym_buf: ["__extend?f?f2".len]u8 = undefined; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(), - .param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "__extend{c}f{c}f2", .{ - floatCompilerRtAbiName(src_bits), - floatCompilerRtAbiName(dst_bits), - }) catch unreachable, - } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); - } - - const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(zcu), 16))); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const vec_len = if (dst_ty.isVector(zcu)) dst_ty.vectorLen(zcu) else 1; - if (src_bits == 16) { - assert(self.hasFeature(.f16c)); - const mat_src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv); - try self.asmRegisterRegister( - .{ .v_ps, .cvtph2 }, - dst_alias, - registerAlias(mat_src_reg, src_abi_size), - ); - switch (dst_bits) { - 32 => {}, - 64 => try self.asmRegisterRegisterRegister( - .{ .v_sd, .cvtss2 }, - dst_alias, - dst_alias, - dst_alias, - ), - else => unreachable, - } - } else { - assert(src_bits == 32 and dst_bits == 64); - if (self.hasFeature(.avx)) switch (vec_len) { - 1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( - .{ .v_sd, .cvtss2 }, - dst_alias, - dst_alias, - try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), - ) else try self.asmRegisterRegisterRegister( - .{ .v_sd, .cvtss2 }, - dst_alias, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), - ), - 2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_pd, .cvtps2 }, - dst_alias, - try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), - ) else try self.asmRegisterRegister( - .{ .v_pd, .cvtps2 }, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), - ), - else => break :result null, - } else if (src_mcv.isBase()) try self.asmRegisterMemory( - switch (vec_len) { - 1 => .{ ._sd, .cvtss2 }, - 2 => .{ ._pd, .cvtps2 }, - else => break :result null, - }, - dst_alias, - try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), - ) else try self.asmRegisterRegister( - switch (vec_len) { - 1 => .{ ._sd, .cvtss2 }, - 2 => .{ ._pd, .cvtps2 }, - else => break :result null, - }, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), - ); - } - break :result dst_mcv; - } orelse return self.fail("TODO implement airFpext from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const src_ty = self.typeOf(ty_op.operand); - const dst_ty = self.typeOfIndex(inst); - - const result = @as(?MCValue, result: { - const src_abi_size: u31 = @intCast(src_ty.abiSize(zcu)); - const dst_abi_size: u31 = @intCast(dst_ty.abiSize(zcu)); - - const src_int_info = src_ty.intInfo(zcu); - const dst_int_info = dst_ty.intInfo(zcu); - const extend = switch (src_int_info.signedness) { - .signed => dst_int_info, - .unsigned => src_int_info, - }.signedness; - - const src_mcv = try self.resolveInst(ty_op.operand); - if (dst_ty.isVector(zcu)) { - const max_abi_size = @max(dst_abi_size, src_abi_size); - const has_avx = self.hasFeature(.avx); - - const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); - const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu); - switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) { - .lt => { - if (max_abi_size > self.vectorSize(.int)) break :result null; - const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) { - else => break :result null, - 1 => switch (src_elem_abi_size) { - else => break :result null, - 2 => switch (dst_int_info.signedness) { - .signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw }, - .unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw }, - }, - }, - 2 => switch (src_elem_abi_size) { - else => break :result null, - 4 => switch (dst_int_info.signedness) { - .signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd }, - .unsigned => if (has_avx) - .{ .vp_w, .ackusd } - else if (self.hasFeature(.sse4_1)) - .{ .p_w, .ackusd } - else - break :result null, - }, - }, - }; - - const dst_mcv: MCValue = if (src_mcv.isRegister() and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else if (has_avx and src_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, dst_abi_size); - - if (has_avx) try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - dst_reg, src_abi_size), - dst_alias, - ) else try self.asmRegisterRegister( - mir_tag, - dst_alias, - dst_alias, - ); - break :result dst_mcv; - }, - .eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - break :result src_mcv - else { - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); - break :result dst_mcv; - }, - .gt => if (self.hasFeature(.sse4_1)) { - if (max_abi_size > self.vectorSize(.int)) break :result null; - const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) { - else => break :result null, - 2 => if (has_avx) .vp_w else .p_w, - 4 => if (has_avx) .vp_d else .p_d, - 8 => if (has_avx) .vp_q else .p_q, - }, switch (src_elem_abi_size) { - else => break :result null, - 1 => switch (extend) { - .signed => .movsxb, - .unsigned => .movzxb, - }, - 2 => switch (extend) { - .signed => .movsxw, - .unsigned => .movzxw, - }, - 4 => switch (extend) { - .signed => .movsxd, - .unsigned => .movzxd, - }, - } }; - - const dst_mcv: MCValue = if (src_mcv.isRegister() and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }; - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, dst_abi_size); - - if (src_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_alias, - try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), - ); - break :result dst_mcv; - } else { - const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) { - else => break :result null, - 2 => switch (src_elem_abi_size) { - else => break :result null, - 1 => .{ .p_, .unpcklbw }, - }, - 4 => switch (src_elem_abi_size) { - else => break :result null, - 2 => .{ .p_, .unpcklwd }, - }, - 8 => switch (src_elem_abi_size) { - else => break :result null, - 2 => .{ .p_, .unpckldq }, - }, - }; - - const dst_mcv: MCValue = if (src_mcv.isRegister() and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); - const dst_reg = dst_mcv.getReg().?; - - const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); - const ext_alias = registerAlias(ext_reg, src_abi_size); - const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg); - defer self.register_manager.unlockReg(ext_lock); - - try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias); - switch (extend) { - .signed => try self.asmRegisterRegister( - .{ switch (src_elem_abi_size) { - else => unreachable, - 1 => .p_b, - 2 => .p_w, - 4 => .p_d, - }, .cmpgt }, - ext_alias, - registerAlias(dst_reg, src_abi_size), - ), - .unsigned => {}, - } - try self.asmRegisterRegister( - mir_tag, - registerAlias(dst_reg, dst_abi_size), - registerAlias(ext_reg, dst_abi_size), - ); - break :result dst_mcv; - }, - } - @compileError("unreachable"); - } - - const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; - - const src_storage_bits: u16 = switch (src_mcv) { - .register, .register_offset => 64, - .register_pair => 128, - .load_frame => |frame_addr| @intCast(self.getFrameAddrSize(frame_addr) * 8), - else => src_int_info.bits, - }; - - const dst_mcv = if ((if (src_mcv.getReg()) |src_reg| src_reg.isClass(.general_purpose) else src_abi_size > 8) and - dst_int_info.bits <= src_storage_bits and - std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable == - std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(min_ty, dst_mcv, src_mcv, .{}); - break :dst dst_mcv; - }; - - if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister()) - .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) } - else - dst_mcv; - - if (dst_mcv.isRegister()) { - try self.truncateRegister(src_ty, dst_mcv.getReg().?); - break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }; - } - - const src_limbs_len = std.math.divCeil(u31, src_abi_size, 8) catch unreachable; - const dst_limbs_len = @divExact(dst_abi_size, 8); - - const high_mcv: MCValue = if (dst_mcv.isBase()) - dst_mcv.address().offset((src_limbs_len - 1) * 8).deref() - else - .{ .register = dst_mcv.register_pair[1] }; - const high_reg = if (high_mcv.isRegister()) - high_mcv.getReg().? - else - try self.copyToTmpRegister(switch (src_int_info.signedness) { - .signed => .isize, - .unsigned => .usize, - }, high_mcv); - const high_lock = self.register_manager.lockRegAssumeUnused(high_reg); - defer self.register_manager.unlockReg(high_lock); - - const high_bits = src_int_info.bits % 64; - if (high_bits > 0) { - try self.truncateRegister(src_ty, high_reg); - const high_ty: Type = if (dst_int_info.bits >= 64) .usize else dst_ty; - try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{}); - } - - if (dst_limbs_len > src_limbs_len) try self.genInlineMemset( - dst_mcv.address().offset(src_limbs_len * 8), - switch (extend) { - .signed => extend: { - const extend_mcv = MCValue{ .register = high_reg }; - try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, extend_mcv, .u8, .{ .immediate = 63 }); - break :extend extend_mcv; - }, - .unsigned => .{ .immediate = 0 }, - }, - .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 }, - .{}, - ); - - break :result dst_mcv; - }) orelse return self.fail("TODO implement airIntCast from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dst_ty = self.typeOfIndex(inst); - const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const src_ty = self.typeOf(ty_op.operand); - const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); - - const result = result: { - const src_mcv = try self.resolveInst(ty_op.operand); - const src_lock = - if (src_mcv.getReg()) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = if (src_mcv.isRegister() and src_mcv.getReg().?.isClass(self.regClassForType(dst_ty)) and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else if (dst_abi_size <= 8) - try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv) - else if (dst_abi_size <= 16 and !dst_ty.isVector(zcu)) dst: { - const dst_regs = - try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp); - const dst_mcv: MCValue = .{ .register_pair = dst_regs }; - const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); - defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); - - try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); - break :dst dst_mcv; - } else dst: { - const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true); - try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); - break :dst dst_mcv; - }; - - if (dst_ty.zigTypeTag(zcu) == .vector) { - assert(src_ty.zigTypeTag(zcu) == .vector and dst_ty.vectorLen(zcu) == src_ty.vectorLen(zcu)); - const dst_elem_ty = dst_ty.childType(zcu); - const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(zcu)); - const src_elem_ty = src_ty.childType(zcu); - const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(zcu)); - - const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) { - 1 => switch (src_elem_abi_size) { - 2 => switch (dst_ty.vectorLen(zcu)) { - 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw }, - 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null, - else => null, - }, - else => null, - }, - 2 => switch (src_elem_abi_size) { - 4 => switch (dst_ty.vectorLen(zcu)) { - 1...4 => if (self.hasFeature(.avx)) - .{ .vp_w, .ackusd } - else if (self.hasFeature(.sse4_1)) - .{ .p_w, .ackusd } - else - null, - 5...8 => if (self.hasFeature(.avx2)) .{ .vp_w, .ackusd } else null, - else => null, - }, - else => null, - }, - else => null, - }) orelse return self.fail("TODO implement airTrunc for {f}", .{dst_ty.fmt(pt)}); - - const dst_info = dst_elem_ty.intInfo(zcu); - const src_info = src_elem_ty.intInfo(zcu); - - const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits)); - - const splat_ty = try pt.vectorType(.{ - .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)), - .child = src_elem_ty.ip_index, - }); - const splat_abi_size: u32 = @intCast(splat_ty.abiSize(zcu)); - - const splat_val = try pt.aggregateSplatValue(splat_ty, mask_val); - - const splat_mcv = try self.lowerValue(splat_val); - const splat_addr_mcv: MCValue = switch (splat_mcv) { - .memory, .indirect, .load_frame => splat_mcv.address(), - else => .{ .register = try self.copyToTmpRegister(.usize, splat_mcv.address()) }, - }; - - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, src_abi_size); - if (self.hasFeature(.avx)) { - try self.asmRegisterRegisterMemory( - .{ .vp_, .@"and" }, - dst_alias, - dst_alias, - try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }), - ); - if (src_abi_size > 16) { - const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); - const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); - defer self.register_manager.unlockReg(temp_lock); - - try self.asmRegisterRegisterImmediate( - .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract }, - registerAlias(temp_reg, dst_abi_size), - dst_alias, - .u(1), - ); - try self.asmRegisterRegisterRegister( - mir_tag, - registerAlias(dst_reg, dst_abi_size), - registerAlias(dst_reg, dst_abi_size), - registerAlias(temp_reg, dst_abi_size), - ); - } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias); - } else { - try self.asmRegisterMemory( - .{ .p_, .@"and" }, - dst_alias, - try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }), - ); - try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias); - } - break :result dst_mcv; - } - - // when truncating a `u16` to `u5`, for example, those top 3 bits in the result - // have to be removed. this only happens if the dst if not a power-of-two size. - if (dst_abi_size <= 8) { - if (self.regExtraBits(dst_ty) > 0) { - try self.truncateRegister(dst_ty, dst_mcv.register.to64()); - } - } else if (dst_abi_size <= 16) { - const dst_info = dst_ty.intInfo(zcu); - const high_ty = try pt.intType(dst_info.signedness, dst_info.bits - 64); - if (self.regExtraBits(high_ty) > 0) { - try self.truncateRegister(high_ty, dst_mcv.register_pair[1].to64()); - } - } - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - - const slice_ty = self.typeOfIndex(inst); - const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu)); - - const ptr_ty = self.typeOf(bin_op.lhs); - try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{}); - - const len_ty = self.typeOf(bin_op.rhs); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(ptr_ty.abiSize(zcu)), - len_ty, - .{ .air_ref = bin_op.rhs }, - .{}, - ); - - const result = MCValue{ .load_frame = .{ .index = frame_index } }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand); - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); - - const dst_ty = self.typeOfIndex(inst); - if (dst_ty.isAbiInt(zcu)) { - const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const bit_size: u32 = @intCast(dst_ty.bitSize(zcu)); - if (abi_size * 8 > bit_size) { - const dst_lock = switch (dst_mcv) { - .register => |dst_reg| self.register_manager.lockRegAssumeUnused(dst_reg), - else => null, - }; - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - if (dst_mcv.isRegister()) { - try self.truncateRegister(dst_ty, dst_mcv.getReg().?); - } else { - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1)); - const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref(); - try self.genSetReg(tmp_reg, hi_ty, hi_mcv, .{}); - try self.truncateRegister(dst_ty, tmp_reg); - try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg }, .{}); - } - } - } - return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 { - const pt = self.pt; - const zcu = pt.zcu; - const air_tag = self.air.instructions.items(.tag); - const air_data = self.air.instructions.items(.data); - - const dst_ty = self.typeOf(dst_air); - const dst_info = dst_ty.intInfo(zcu); - if (dst_air.toIndex()) |inst| { - switch (air_tag[@intFromEnum(inst)]) { - .intcast => { - const src_ty = self.typeOf(air_data[@intFromEnum(inst)].ty_op.operand); - const src_info = src_ty.intInfo(zcu); - return @min(switch (src_info.signedness) { - .signed => switch (dst_info.signedness) { - .signed => src_info.bits, - .unsigned => src_info.bits - 1, - }, - .unsigned => switch (dst_info.signedness) { - .signed => src_info.bits + 1, - .unsigned => src_info.bits, - }, - }, dst_info.bits); - }, - else => {}, - } - } else if (dst_air.toInterned()) |ip_index| { - var space: Value.BigIntSpace = undefined; - const src_int = Value.fromInterned(ip_index).toBigInt(&space, zcu); - return @as(u16, @intCast(src_int.bitCountTwosComp())) + - @intFromBool(src_int.positive and dst_info.signedness == .signed); - } - return dst_info.bits; -} - -fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const result = result: { - const dst_ty = self.typeOfIndex(inst); - switch (dst_ty.zigTypeTag(zcu)) { - .float, .vector => break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs), - else => {}, - } - const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - - const dst_info = dst_ty.intInfo(zcu); - const src_ty = try pt.intType(dst_info.signedness, switch (tag) { - else => unreachable, - .mul, .mul_wrap => @max( - self.activeIntBits(bin_op.lhs), - self.activeIntBits(bin_op.rhs), - dst_info.bits / 2, - ), - .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_info.bits, - }); - const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); - - if (dst_abi_size == 16 and src_abi_size == 16) switch (tag) { - else => unreachable, - .mul, .mul_wrap => {}, - .div_trunc, .div_floor, .div_exact, .rem, .mod => { - const signed = dst_ty.isSignedInt(zcu); - var sym_buf: ["__udiv?i3".len]u8 = undefined; - const signed_div_floor_state: struct { - frame_index: FrameIndex, - state: State, - reloc: Mir.Inst.Index, - } = if (signed and tag == .div_floor) state: { - const frame_index = try self.allocFrameIndex(.initType(.usize, zcu)); - try self.asmMemoryImmediate( - .{ ._, .mov }, - .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, - .u(0), - ); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const mat_lhs_mcv = switch (lhs_mcv) { - .load_nav, .load_uav, .load_lazy_sym => mat_lhs_mcv: { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address()); - break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => lhs_mcv, - }; - const mat_lhs_lock = switch (mat_lhs_mcv) { - .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), - else => null, - }; - defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); - if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .mov }, - tmp_reg, - try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .mov }, - tmp_reg, - mat_lhs_mcv.register_pair[1], - ); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const mat_rhs_mcv = switch (rhs_mcv) { - .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); - break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => rhs_mcv, - }; - const mat_rhs_lock = switch (mat_rhs_mcv) { - .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), - else => null, - }; - defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); - if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .xor }, - tmp_reg, - try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .xor }, - tmp_reg, - mat_rhs_mcv.register_pair[1], - ); - const state = try self.saveState(); - const reloc = try self.asmJccReloc(.ns, undefined); - - break :state .{ .frame_index = frame_index, .state = state, .reloc = reloc }; - } else undefined; - const call_mcv = try self.genCall( - .{ .extern_func = .{ - .return_type = dst_ty.toIntern(), - .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() }, - .sym = std.fmt.bufPrint(&sym_buf, "__{s}{s}{c}i3", .{ - if (signed) "" else "u", - switch (tag) { - .div_trunc, .div_exact => "div", - .div_floor => if (signed) "mod" else "div", - .rem, .mod => "mod", - else => unreachable, - }, - intCompilerRtAbiName(@intCast(dst_ty.bitSize(zcu))), - }) catch unreachable, - } }, - &.{ src_ty, src_ty }, - &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, - .{}, - ); - break :result if (signed) switch (tag) { - .div_floor => { - try self.asmRegisterRegister( - .{ ._, .@"or" }, - call_mcv.register_pair[0], - call_mcv.register_pair[1], - ); - try self.asmSetccMemory(.nz, .{ - .base = .{ .frame = signed_div_floor_state.frame_index }, - .mod = .{ .rm = .{ .size = .byte } }, - }); - try self.restoreState(signed_div_floor_state.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - self.performReloc(signed_div_floor_state.reloc); - const dst_mcv = try self.genCall( - .{ .extern_func = .{ - .return_type = dst_ty.toIntern(), - .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() }, - .sym = std.fmt.bufPrint(&sym_buf, "__div{c}i3", .{ - intCompilerRtAbiName(@intCast(dst_ty.bitSize(zcu))), - }) catch unreachable, - } }, - &.{ src_ty, src_ty }, - &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, - .{}, - ); - try self.asmRegisterMemory( - .{ ._, .sub }, - dst_mcv.register_pair[0], - .{ - .base = .{ .frame = signed_div_floor_state.frame_index }, - .mod = .{ .rm = .{ .size = .qword } }, - }, - ); - try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0)); - try self.freeValue( - .{ .load_frame = .{ .index = signed_div_floor_state.frame_index } }, - ); - break :result dst_mcv; - }, - .mod => { - const dst_regs = call_mcv.register_pair; - const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); - defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); - - const tmp_regs = - try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); - const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); - defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const mat_rhs_mcv = switch (rhs_mcv) { - .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); - break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => rhs_mcv, - }; - const mat_rhs_lock = switch (mat_rhs_mcv) { - .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), - else => null, - }; - defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); - - for (tmp_regs, dst_regs) |tmp_reg, dst_reg| - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg); - if (mat_rhs_mcv.isBase()) { - try self.asmRegisterMemory( - .{ ._, .add }, - tmp_regs[0], - try mat_rhs_mcv.mem(self, .{ .size = .qword }), - ); - try self.asmRegisterMemory( - .{ ._, .adc }, - tmp_regs[1], - try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ); - } else for ( - [_]Mir.Inst.Tag{ .add, .adc }, - tmp_regs, - mat_rhs_mcv.register_pair, - ) |op, tmp_reg, rhs_reg| - try self.asmRegisterRegister(.{ ._, op }, tmp_reg, rhs_reg); - try self.asmRegisterRegister(.{ ._, .@"test" }, dst_regs[1], dst_regs[1]); - for (dst_regs, tmp_regs) |dst_reg, tmp_reg| - try self.asmCmovccRegisterRegister(.s, dst_reg, tmp_reg); - break :result call_mcv; - }, - else => call_mcv, - } else call_mcv; - }, - }; - - try self.spillEflagsIfOccupied(); - try self.spillRegisters(&.{ .rax, .rcx, .rdx }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const rhs_mcv = try self.resolveInst(bin_op.rhs); - break :result try self.genMulDivBinOp(tag, inst, dst_ty, src_ty, lhs_mcv, rhs_mcv); - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ty = self.typeOf(bin_op.lhs); - if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail( - "TODO implement airAddSat for {f}", - .{ty.fmt(pt)}, - ); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) - lhs_mcv - else - try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv); - const dst_reg = dst_mcv.register; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const rhs_lock = switch (rhs_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const limit_mcv = MCValue{ .register = limit_reg }; - const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg); - defer self.register_manager.unlockReg(limit_lock); - - const reg_bits = self.regBitSize(ty); - const reg_extra_bits = self.regExtraBits(ty); - const cc: Condition = if (ty.isSignedInt(zcu)) cc: { - if (reg_extra_bits > 0) { - try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); - } - try self.genSetReg(limit_reg, ty, dst_mcv, .{}); - try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 }); - try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{ - .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1, - }); - if (reg_extra_bits > 0) { - const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv); - const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg }; - const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg); - defer self.register_manager.unlockReg(shifted_rhs_lock); - - try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits }); - try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv); - } else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv); - break :cc .o; - } else cc: { - try self.genSetReg(limit_reg, ty, .{ - .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)), - }, .{}); - - try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv); - if (reg_extra_bits > 0) { - try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, limit_mcv); - break :cc .a; - } - break :cc .c; - }; - - const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); - try self.asmCmovccRegisterRegister( - cc, - registerAlias(dst_reg, cmov_abi_size), - registerAlias(limit_reg, cmov_abi_size), - ); - - if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) - try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); - - return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ty = self.typeOf(bin_op.lhs); - if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail( - "TODO implement airSubSat for {f}", - .{ty.fmt(pt)}, - ); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) - lhs_mcv - else - try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv); - const dst_reg = dst_mcv.register; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const rhs_lock = switch (rhs_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const limit_mcv = MCValue{ .register = limit_reg }; - const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg); - defer self.register_manager.unlockReg(limit_lock); - - const reg_bits = self.regBitSize(ty); - const reg_extra_bits = self.regExtraBits(ty); - const cc: Condition = if (ty.isSignedInt(zcu)) cc: { - if (reg_extra_bits > 0) { - try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); - } - try self.genSetReg(limit_reg, ty, dst_mcv, .{}); - try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 }); - try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{ - .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1, - }); - if (reg_extra_bits > 0) { - const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv); - const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg }; - const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg); - defer self.register_manager.unlockReg(shifted_rhs_lock); - - try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits }); - try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv); - } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv); - break :cc .o; - } else cc: { - try self.genSetReg(limit_reg, ty, .{ .immediate = 0 }, .{}); - try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv); - break :cc .c; - }; - - const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); - try self.asmCmovccRegisterRegister( - cc, - registerAlias(dst_reg, cmov_abi_size), - registerAlias(limit_reg, cmov_abi_size), - ); - - if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) - try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); - - return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ty = self.typeOf(bin_op.lhs); - - const result = result: { - if (ty.toIntern() == .i128_type) { - const ptr_c_int = try pt.singleMutPtrType(.c_int); - const overflow = try self.allocTempRegOrMem(.c_int, false); - - const dst_mcv = try self.genCall(.{ .extern_func = .{ - .return_type = .i128_type, - .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() }, - .sym = "__muloti4", - } }, &.{ .i128, .i128, ptr_c_int }, &.{ - .{ .air_ref = bin_op.lhs }, - .{ .air_ref = bin_op.rhs }, - overflow.address(), - }, .{}); - const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair); - defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const mat_lhs_mcv = switch (lhs_mcv) { - .load_nav, .load_uav, .load_lazy_sym => mat_lhs_mcv: { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address()); - break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => lhs_mcv, - }; - const mat_lhs_lock = switch (mat_lhs_mcv) { - .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), - else => null, - }; - defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); - if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .mov }, - tmp_reg, - try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .mov }, - tmp_reg, - mat_lhs_mcv.register_pair[1], - ); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const mat_rhs_mcv = switch (rhs_mcv) { - .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); - break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => rhs_mcv, - }; - const mat_rhs_lock = switch (mat_rhs_mcv) { - .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), - else => null, - }; - defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); - if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .xor }, - tmp_reg, - try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .xor }, - tmp_reg, - mat_rhs_mcv.register_pair[1], - ); - - try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63)); - try self.asmRegister(.{ ._, .not }, tmp_reg); - try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0)); - try self.freeValue(overflow); - try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg); - try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63)); - try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg); - break :result dst_mcv; - } - - if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail( - "TODO implement airMulSat for {f}", - .{ty.fmt(pt)}, - ); - - try self.spillRegisters(&.{ .rax, .rcx, .rdx }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const lhs_lock = switch (lhs_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const rhs_lock = switch (rhs_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const limit_mcv = MCValue{ .register = limit_reg }; - const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg); - defer self.register_manager.unlockReg(limit_lock); - - const reg_bits = self.regBitSize(ty); - const cc: Condition = if (ty.isSignedInt(zcu)) cc: { - try self.genSetReg(limit_reg, ty, lhs_mcv, .{}); - try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv); - try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 }); - try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{ - .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1, - }); - break :cc .o; - } else cc: { - try self.genSetReg(limit_reg, ty, .{ - .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits), - }, .{}); - break :cc .c; - }; - - const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv); - const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); - try self.asmCmovccRegisterRegister( - cc, - registerAlias(dst_mcv.register, cmov_abi_size), - registerAlias(limit_reg, cmov_abi_size), - ); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const result: MCValue = result: { - const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; - const ty = self.typeOf(bin_op.lhs); - switch (ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO implement add/sub with overflow for Vector type", .{}), - .int => { - try self.spillEflagsIfOccupied(); - try self.spillRegisters(&.{ .rcx, .rdi, .rsi }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rcx, .rdi, .rsi }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const partial_mcv = try self.genBinOp(null, switch (tag) { - .add_with_overflow => .add, - .sub_with_overflow => .sub, - else => unreachable, - }, bin_op.lhs, bin_op.rhs); - const int_info = ty.intInfo(zcu); - const cc: Condition = switch (int_info.signedness) { - .unsigned => .c, - .signed => .o, - }; - - const tuple_ty = self.typeOfIndex(inst); - if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) { - switch (partial_mcv) { - .register => |reg| { - self.eflags_inst = inst; - break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; - }, - else => {}, - } - - const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(1, zcu)), - .u1, - .{ .eflags = cc }, - .{}, - ); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(0, zcu)), - ty, - partial_mcv, - .{}, - ); - break :result .{ .load_frame = .{ .index = frame_index } }; - } - - const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); - try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - else => unreachable, - } - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const result: MCValue = result: { - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - switch (lhs_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO implement shl with overflow for Vector type", .{}), - .int => { - try self.spillEflagsIfOccupied(); - try self.spillRegisters(&.{ .rcx, .rdi, .rsi }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rcx, .rdi, .rsi }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const int_info = lhs_ty.intInfo(zcu); - - const partial_mcv = try self.genShiftBinOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty); - const partial_lock = switch (partial_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (partial_lock) |lock| self.register_manager.unlockReg(lock); - - const tmp_mcv = try self.genShiftBinOp(.shr, null, partial_mcv, rhs, lhs_ty, rhs_ty); - const tmp_lock = switch (tmp_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs); - const cc = Condition.ne; - - const tuple_ty = self.typeOfIndex(inst); - if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) { - switch (partial_mcv) { - .register => |reg| { - self.eflags_inst = inst; - break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; - }, - else => {}, - } - - const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(1, zcu)), - tuple_ty.fieldType(1, zcu), - .{ .eflags = cc }, - .{}, - ); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(0, zcu)), - tuple_ty.fieldType(0, zcu), - partial_mcv, - .{}, - ); - break :result .{ .load_frame = .{ .index = frame_index } }; - } - - const frame_index = - try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); - try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - else => unreachable, - } - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn genSetFrameTruncatedOverflowCompare( - self: *CodeGen, - tuple_ty: Type, - frame_index: FrameIndex, - src_mcv: MCValue, - overflow_cc: ?Condition, -) !void { - const pt = self.pt; - const zcu = pt.zcu; - const src_lock = switch (src_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const ty = tuple_ty.fieldType(0, zcu); - const ty_size = ty.abiSize(zcu); - const int_info = ty.intInfo(zcu); - - const hi_bits = (int_info.bits - 1) % 64 + 1; - const hi_ty = try pt.intType(int_info.signedness, hi_bits); - - const limb_bits: u16 = @intCast(if (int_info.bits <= 64) self.regBitSize(ty) else 64); - const limb_ty = try pt.intType(int_info.signedness, limb_bits); - - const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits); - - const temp_regs = - try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp); - const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); - defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); - - const overflow_reg = temp_regs[0]; - if (overflow_cc) |cc| try self.asmSetccRegister(cc, overflow_reg.to8()); - - const scratch_reg = temp_regs[1]; - const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8; - const hi_limb_mcv = if (hi_limb_off > 0) - src_mcv.address().offset(int_info.bits / 64 * 8).deref() - else - src_mcv; - try self.genSetReg(scratch_reg, limb_ty, hi_limb_mcv, .{}); - try self.truncateRegister(hi_ty, scratch_reg); - try self.genBinOpMir(.{ ._, .cmp }, limb_ty, .{ .register = scratch_reg }, hi_limb_mcv); - - const eq_reg = temp_regs[2]; - if (overflow_cc) |_| { - try self.asmSetccRegister(.ne, eq_reg.to8()); - try self.genBinOpMir(.{ ._, .@"or" }, .u8, .{ .register = overflow_reg }, .{ .register = eq_reg }); - } - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(1, zcu)), - tuple_ty.fieldType(1, zcu), - if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne }, - .{}, - ); - - const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu)); - if (hi_limb_off > 0) try self.genSetMem( - .{ .frame = frame_index }, - payload_off, - rest_ty, - src_mcv, - .{}, - ); - try self.genSetMem( - .{ .frame = frame_index }, - payload_off + hi_limb_off, - limb_ty, - .{ .register = scratch_reg }, - .{}, - ); - var ext_off: i32 = hi_limb_off + 8; - if (ext_off < ty_size) { - switch (int_info.signedness) { - .signed => try self.asmRegisterImmediate(.{ ._r, .sa }, scratch_reg.to64(), .s(63)), - .unsigned => try self.asmRegisterRegister(.{ ._, .xor }, scratch_reg.to32(), scratch_reg.to32()), - } - while (ext_off < ty_size) : (ext_off += 8) try self.genSetMem( - .{ .frame = frame_index }, - payload_off + ext_off, - limb_ty, - .{ .register = scratch_reg }, - .{}, - ); - } -} - -fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const tuple_ty = self.typeOfIndex(inst); - const dst_ty = self.typeOf(bin_op.lhs); - const result: MCValue = switch (dst_ty.zigTypeTag(zcu)) { - .vector => return self.fail("TODO implement airMulWithOverflow for {f}", .{dst_ty.fmt(pt)}), - .int => result: { - const dst_info = dst_ty.intInfo(zcu); - if (dst_info.bits > 128 and dst_info.signedness == .unsigned) { - const slow_inc = self.hasFeature(.slow_incdec); - const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; - - try self.spillRegisters(&.{ .rax, .rcx, .rdx }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genInlineMemset( - dst_mcv.address(), - .{ .immediate = 0 }, - .{ .immediate = tuple_ty.abiSize(zcu) }, - .{}, - ); - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const rhs_mcv = try self.resolveInst(bin_op.rhs); - - const temp_regs = - try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); - const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs); - defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); - - try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32()); - - const outer_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[1].to64(), .{ - .base = .{ .frame = rhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[0].to64(), - .scale = .@"8", - .disp = rhs_mcv.load_frame.off, - } }, - }); - try self.asmRegisterRegister(.{ ._, .@"test" }, temp_regs[1].to64(), temp_regs[1].to64()); - const skip_inner = try self.asmJccReloc(.z, undefined); - - try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[2].to32(), temp_regs[2].to32()); - try self.asmRegisterRegister(.{ ._, .mov }, temp_regs[3].to32(), temp_regs[0].to32()); - try self.asmRegisterRegister(.{ ._, .xor }, .ecx, .ecx); - try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx); - - const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1)); - try self.asmMemoryRegister(.{ ._, .adc }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[3].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off + - @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), - } }, - }, .rdx); - try self.asmSetccRegister(.c, .cl); - - try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[2].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off, - } }, - }); - try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64()); - - try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1)); - try self.asmMemoryRegister(.{ ._, .adc }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[3].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off + - @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), - } }, - }, .rax); - try self.asmSetccRegister(.c, .ch); - - if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, temp_regs[2].to32()); - try self.asmRegister(.{ ._c, .in }, temp_regs[3].to32()); - } - try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len)); - _ = try self.asmJccReloc(.b, inner_loop); - - try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx); - const overflow = try self.asmJccReloc(.nz, undefined); - const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len)); - const no_overflow = try self.asmJccReloc(.nb, undefined); - if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, temp_regs[2].to32()); - } - try self.asmMemoryImmediate(.{ ._, .cmp }, .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[2].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off - 8, - } }, - }, .u(0)); - _ = try self.asmJccReloc(.z, overflow_loop); - self.performReloc(overflow); - try self.asmMemoryImmediate(.{ ._, .mov }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .byte, - .disp = dst_mcv.load_frame.off + - @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), - } }, - }, .u(1)); - self.performReloc(no_overflow); - - self.performReloc(skip_inner); - if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32()); - } - try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len)); - _ = try self.asmJccReloc(.b, outer_loop); - - break :result dst_mcv; - } - - const lhs_active_bits = self.activeIntBits(bin_op.lhs); - const rhs_active_bits = self.activeIntBits(bin_op.rhs); - const src_bits = @max(lhs_active_bits, rhs_active_bits, dst_info.bits / 2); - const src_ty = try pt.intType(dst_info.signedness, src_bits); - if (src_bits > 64 and src_bits <= 128 and - dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) { - .signed => { - const ptr_c_int = try pt.singleMutPtrType(.c_int); - const overflow = try self.allocTempRegOrMem(.c_int, false); - const result = try self.genCall(.{ .extern_func = .{ - .return_type = .i128_type, - .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() }, - .sym = "__muloti4", - } }, &.{ .i128, .i128, ptr_c_int }, &.{ - .{ .air_ref = bin_op.lhs }, - .{ .air_ref = bin_op.rhs }, - overflow.address(), - }, .{}); - - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genSetMem( - .{ .frame = dst_mcv.load_frame.index }, - @intCast(tuple_ty.structFieldOffset(0, zcu)), - tuple_ty.fieldType(0, zcu), - result, - .{}, - ); - try self.asmMemoryImmediate( - .{ ._, .cmp }, - try overflow.mem(self, .{ .size = self.memSize(.c_int) }), - .s(0), - ); - try self.genSetMem( - .{ .frame = dst_mcv.load_frame.index }, - @intCast(tuple_ty.structFieldOffset(1, zcu)), - tuple_ty.fieldType(1, zcu), - .{ .eflags = .ne }, - .{}, - ); - try self.freeValue(overflow); - break :result dst_mcv; - }, - .unsigned => { - try self.spillEflagsIfOccupied(); - try self.spillRegisters(&.{ .rax, .rdx }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const tmp_regs = - try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); - const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs); - defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const mat_lhs_mcv = mat_lhs_mcv: switch (lhs_mcv) { - .register => |lhs_reg| switch (lhs_reg.class()) { - else => lhs_mcv, - .sse => { - const mat_lhs_mcv: MCValue = .{ - .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp), - }; - try self.genCopy(dst_ty, mat_lhs_mcv, lhs_mcv, .{}); - break :mat_lhs_mcv mat_lhs_mcv; - }, - }, - .load_nav, .load_uav, .load_lazy_sym => { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address()); - break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => lhs_mcv, - }; - const mat_lhs_locks: [2]?RegisterLock = switch (mat_lhs_mcv) { - .register_pair => |mat_lhs_regs| self.register_manager.lockRegs(2, mat_lhs_regs), - .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null }, - else => @splat(null), - }; - defer for (mat_lhs_locks) |mat_lhs_lock| if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); - const mat_rhs_mcv = mat_rhs_mcv: switch (rhs_mcv) { - .register => |rhs_reg| switch (rhs_reg.class()) { - else => rhs_mcv, - .sse => { - const mat_rhs_mcv: MCValue = .{ - .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp), - }; - try self.genCopy(dst_ty, mat_rhs_mcv, rhs_mcv, .{}); - break :mat_rhs_mcv mat_rhs_mcv; - }, - }, - .load_nav, .load_uav, .load_lazy_sym => { - // TODO clean this up! - const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); - break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; - }, - else => rhs_mcv, - }; - const mat_rhs_locks: [2]?RegisterLock = switch (mat_rhs_mcv) { - .register_pair => |mat_rhs_regs| self.register_manager.lockRegs(2, mat_rhs_regs), - .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null }, - else => @splat(null), - }; - defer for (mat_rhs_locks) |mat_rhs_lock| if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); - - if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .mov }, - .rax, - try mat_lhs_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .mov }, - .rax, - mat_lhs_mcv.register_pair[0], - ); - if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .mov }, - tmp_regs[0], - try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .mov }, - tmp_regs[0], - mat_rhs_mcv.register_pair[1], - ); - try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_regs[0], tmp_regs[0]); - try self.asmSetccRegister(.nz, tmp_regs[1].to8()); - try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax); - try self.asmSetccRegister(.o, tmp_regs[2].to8()); - if (mat_rhs_mcv.isBase()) - try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) - else - try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); - try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); - try self.asmSetccRegister(.c, tmp_regs[3].to8()); - try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8()); - if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .mov }, - tmp_regs[0], - try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._, .mov }, - tmp_regs[0], - mat_lhs_mcv.register_pair[1], - ); - try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_regs[0], tmp_regs[0]); - try self.asmSetccRegister(.nz, tmp_regs[3].to8()); - try self.asmRegisterRegister( - .{ ._, .@"and" }, - tmp_regs[1].to8(), - tmp_regs[3].to8(), - ); - try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); - if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( - .{ .i_, .mul }, - tmp_regs[0], - try mat_rhs_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ .i_, .mul }, - tmp_regs[0], - mat_rhs_mcv.register_pair[0], - ); - try self.asmSetccRegister(.o, tmp_regs[2].to8()); - try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); - try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); - try self.asmSetccRegister(.c, tmp_regs[2].to8()); - try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); - - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genSetMem( - .{ .frame = dst_mcv.load_frame.index }, - @intCast(tuple_ty.structFieldOffset(0, zcu)), - tuple_ty.fieldType(0, zcu), - .{ .register_pair = .{ .rax, .rdx } }, - .{}, - ); - try self.genSetMem( - .{ .frame = dst_mcv.load_frame.index }, - @intCast(tuple_ty.structFieldOffset(1, zcu)), - tuple_ty.fieldType(1, zcu), - .{ .register = tmp_regs[1] }, - .{}, - ); - break :result dst_mcv; - }, - }; - - try self.spillEflagsIfOccupied(); - try self.spillRegisters(&.{ .rax, .rcx, .rdx, .rdi, .rsi }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(5, .{ .rax, .rcx, .rdx, .rdi, .rsi }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const cc: Condition = switch (dst_info.signedness) { - .unsigned => .c, - .signed => .o, - }; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const extra_bits = if (dst_info.bits <= 64) - self.regExtraBits(dst_ty) - else - dst_info.bits % 64; - const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs); - - switch (partial_mcv) { - .register => |reg| if (extra_bits == 0) { - self.eflags_inst = inst; - break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; - } else { - const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); - try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - else => { - // For now, this is the only supported multiply that doesn't fit in a register. - if (dst_info.bits > 128 or src_bits != 64) - return self.fail("TODO implement airWithOverflow from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - - const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); - if (dst_info.bits >= lhs_active_bits + rhs_active_bits) { - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(0, zcu)), - tuple_ty.fieldType(0, zcu), - partial_mcv, - .{}, - ); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(tuple_ty.structFieldOffset(1, zcu)), - tuple_ty.fieldType(1, zcu), - .{ .immediate = 0 }, // cc being set is impossible - .{}, - ); - } else try self.genSetFrameTruncatedOverflowCompare( - tuple_ty, - frame_index, - partial_mcv, - null, - ); - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - } - }, - else => unreachable, - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -/// Generates signed or unsigned integer multiplication/division. -/// Clobbers .rax and .rdx registers. -/// Quotient is saved in .rax and remainder in .rdx. -fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void { - const pt = self.pt; - const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); - const bit_size: u32 = @intCast(self.regBitSize(ty)); - if (abi_size > 8) { - return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{}); - } - - try self.genSetReg(.rax, ty, lhs, .{}); - switch (tag[1]) { - else => unreachable, - .mul => {}, - .div => switch (tag[0]) { - ._ => { - const hi_reg: Register = - switch (bit_size) { - 8 => .ah, - 16, 32, 64 => .edx, - else => unreachable, - }; - try self.asmRegisterRegister(.{ ._, .xor }, hi_reg, hi_reg); - }, - .i_ => try self.asmOpOnly(.{ ._, switch (bit_size) { - 8 => .cbw, - 16 => .cwd, - 32 => .cdq, - 64 => .cqo, - else => unreachable, - } }), - else => unreachable, - }, - } - - const mat_rhs: MCValue = switch (rhs) { - .register, .indirect, .load_frame => rhs, - else => .{ .register = try self.copyToTmpRegister(ty, rhs) }, - }; - switch (mat_rhs) { - .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)), - .memory, .indirect, .load_frame => try self.asmMemory( - tag, - try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }), - ), - else => unreachable, - } - if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah); -} - -/// Always returns a register. -/// Clobbers .rax and .rdx registers. -fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const int_info = ty.intInfo(zcu); - const dividend = switch (lhs) { - .register => |reg| reg, - else => try self.copyToTmpRegister(ty, lhs), - }; - const dividend_lock = self.register_manager.lockReg(dividend); - defer if (dividend_lock) |lock| self.register_manager.unlockReg(lock); - - const divisor = switch (rhs) { - .register => |reg| reg, - else => try self.copyToTmpRegister(ty, rhs), - }; - const divisor_lock = self.register_manager.lockReg(divisor); - defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genIntMulDivOpMir( - switch (int_info.signedness) { - .signed => .{ .i_, .div }, - .unsigned => .{ ._, .div }, - }, - ty, - .{ .register = dividend }, - .{ .register = divisor }, - ); - - try self.asmRegisterRegister( - .{ ._, .xor }, - registerAlias(divisor, abi_size), - registerAlias(dividend, abi_size), - ); - try self.asmRegisterImmediate( - .{ ._r, .sa }, - registerAlias(divisor, abi_size), - .u(int_info.bits - 1), - ); - try self.asmRegisterRegister( - .{ ._, .@"test" }, - registerAlias(.rdx, abi_size), - registerAlias(.rdx, abi_size), - ); - try self.asmCmovccRegisterRegister( - .z, - registerAlias(divisor, @max(abi_size, 2)), - registerAlias(.rdx, @max(abi_size, 2)), - ); - try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax }); - return MCValue{ .register = divisor }; -} - -fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const air_tags = self.air.instructions.items(.tag); - const tag = air_tags[@intFromEnum(inst)]; - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - const result: MCValue = result: { - switch (lhs_ty.zigTypeTag(zcu)) { - .int => { - try self.spillRegisters(&.{.rcx}); - try self.register_manager.getKnownReg(.rcx, null); - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const rhs_mcv = try self.resolveInst(bin_op.rhs); - - const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty); - switch (tag) { - .shr, .shr_exact, .shl_exact => {}, - .shl => switch (dst_mcv) { - .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg), - .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]), - .load_frame => |frame_addr| { - const tmp_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu)); - const tmp_ty: Type = if (lhs_bits > 64) .usize else lhs_ty; - const off = frame_addr.off + (lhs_bits - 1) / 64 * 8; - try self.genSetReg( - tmp_reg, - tmp_ty, - .{ .load_frame = .{ .index = frame_addr.index, .off = off } }, - .{}, - ); - try self.truncateRegister(lhs_ty, tmp_reg); - try self.genSetMem( - .{ .frame = frame_addr.index }, - off, - tmp_ty, - .{ .register = tmp_reg }, - .{}, - ); - }, - else => {}, - }, - else => unreachable, - } - break :result dst_mcv; - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(zcu).intInfo(zcu).bits) { - else => null, - 16 => switch (lhs_ty.vectorLen(zcu)) { - else => null, - 1...8 => switch (tag) { - else => unreachable, - .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_w, .sra } - else - .{ .p_w, .sra }, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_w, .srl } - else - .{ .p_w, .srl }, - }, - .shl, .shl_exact => if (self.hasFeature(.avx)) - .{ .vp_w, .sll } - else - .{ .p_w, .sll }, - }, - 9...16 => switch (tag) { - else => unreachable, - .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null, - .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null, - }, - .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null, - }, - }, - 32 => switch (lhs_ty.vectorLen(zcu)) { - else => null, - 1...4 => switch (tag) { - else => unreachable, - .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_d, .sra } - else - .{ .p_d, .sra }, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_d, .srl } - else - .{ .p_d, .srl }, - }, - .shl, .shl_exact => if (self.hasFeature(.avx)) - .{ .vp_d, .sll } - else - .{ .p_d, .sll }, - }, - 5...8 => switch (tag) { - else => unreachable, - .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null, - .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null, - }, - .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null, - }, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - else => null, - 1...2 => switch (tag) { - else => unreachable, - .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_q, .sra } - else - .{ .p_q, .sra }, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_q, .srl } - else - .{ .p_q, .srl }, - }, - .shl, .shl_exact => if (self.hasFeature(.avx)) - .{ .vp_q, .sll } - else - .{ .p_q, .sll }, - }, - 3...4 => switch (tag) { - else => unreachable, - .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null, - .unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null, - }, - .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null, - }, - }, - })) |mir_tag| if (try self.air.value(bin_op.rhs, pt)) |rhs_val| { - switch (zcu.intern_pool.indexToKey(rhs_val.toIntern())) { - .aggregate => |rhs_aggregate| switch (rhs_aggregate.storage) { - .repeated_elem => |rhs_elem| { - const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) - .{lhs_mcv.getReg().?} ** 2 - else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{ - try self.register_manager.allocReg(inst, abi.RegisterClass.sse), - lhs_mcv.getReg().?, - } else .{(try self.copyToRegisterWithInstTracking( - inst, - lhs_ty, - lhs_mcv, - )).register} ** 2; - const reg_locks = - self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg }); - defer for (reg_locks) |reg_lock| if (reg_lock) |lock| - self.register_manager.unlockReg(lock); - - const shift_imm: Immediate = - .u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu))); - if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate( - mir_tag, - registerAlias(dst_reg, abi_size), - registerAlias(lhs_reg, abi_size), - shift_imm, - ) else { - assert(dst_reg.id() == lhs_reg.id()); - try self.asmRegisterImmediate( - mir_tag, - registerAlias(dst_reg, abi_size), - shift_imm, - ); - } - break :result .{ .register = dst_reg }; - }, - else => {}, - }, - else => {}, - } - } else if (bin_op.rhs.toIndex()) |rhs_inst| switch (air_tags[@intFromEnum(rhs_inst)]) { - .splat => { - const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) - .{lhs_mcv.getReg().?} ** 2 - else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{ - try self.register_manager.allocReg(inst, abi.RegisterClass.sse), - lhs_mcv.getReg().?, - } else .{(try self.copyToRegisterWithInstTracking( - inst, - lhs_ty, - lhs_mcv, - )).register} ** 2; - const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg }); - defer for (reg_locks) |reg_lock| if (reg_lock) |lock| - self.register_manager.unlockReg(lock); - - const shift_reg = - try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs }); - const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg); - defer self.register_manager.unlockReg(shift_lock); - - const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type }); - const mask_mcv = try self.lowerValue(try pt.aggregateValue( - mask_ty, - &([1]InternPool.Index{ - (try rhs_ty.childType(zcu).maxIntScalar(pt, .u8)).toIntern(), - } ++ [1]InternPool.Index{.zero_u8} ** 15), - )); - const mask_addr_reg = try self.copyToTmpRegister(.usize, mask_mcv.address()); - const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg); - defer self.register_manager.unlockReg(mask_addr_lock); - - if (self.hasFeature(.avx)) { - try self.asmRegisterRegisterMemory( - .{ .vp_, .@"and" }, - shift_reg.to128(), - shift_reg.to128(), - .{ - .base = .{ .reg = mask_addr_reg }, - .mod = .{ .rm = .{ .size = .xword } }, - }, - ); - try self.asmRegisterRegisterRegister( - mir_tag, - registerAlias(dst_reg, abi_size), - registerAlias(lhs_reg, abi_size), - shift_reg.to128(), - ); - } else { - try self.asmRegisterMemory( - .{ .p_, .@"and" }, - shift_reg.to128(), - .{ - .base = .{ .reg = mask_addr_reg }, - .mod = .{ .rm = .{ .size = .xword } }, - }, - ); - assert(dst_reg.id() == lhs_reg.id()); - try self.asmRegisterRegister( - mir_tag, - registerAlias(dst_reg, abi_size), - shift_reg.to128(), - ); - } - break :result .{ .register = dst_reg }; - }, - else => {}, - }, - else => {}, - }, - else => {}, - } - return self.fail("TODO implement airShlShrBinOp for {f}", .{lhs_ty.fmt(pt)}); - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs_ty = self.typeOf(bin_op.lhs); - const rhs_ty = self.typeOf(bin_op.rhs); - - const result: MCValue = result: { - switch (lhs_ty.zigTypeTag(zcu)) { - .int => { - const lhs_bits = lhs_ty.bitSize(zcu); - const rhs_bits = rhs_ty.bitSize(zcu); - if (!(lhs_bits <= 32 and rhs_bits <= 5) and !(lhs_bits > 32 and lhs_bits <= 64 and rhs_bits <= 6) and !(rhs_bits <= std.math.log2(lhs_bits))) { - return self.fail("TODO implement shl_sat for {} with lhs bits {}, rhs bits {}", .{ self.target.cpu.arch, lhs_bits, rhs_bits }); - } - - // clobberred by genShiftBinOp - try self.spillRegisters(&.{.rcx}); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - var lhs_temp1 = try self.tempInit(lhs_ty, lhs_mcv); - const rhs_mcv = try self.resolveInst(bin_op.rhs); - - const lhs_lock = switch (lhs_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - // shift left - const dst_mcv = try self.genShiftBinOp(.shl, null, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty); - switch (dst_mcv) { - .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg), - .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]), - .load_frame => |frame_addr| { - const tmp_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const lhs_bits_u31: u31 = @intCast(lhs_bits); - const tmp_ty: Type = if (lhs_bits_u31 > 64) .usize else lhs_ty; - const off = frame_addr.off + (lhs_bits_u31 - 1) / 64 * 8; - try self.genSetReg( - tmp_reg, - tmp_ty, - .{ .load_frame = .{ .index = frame_addr.index, .off = off } }, - .{}, - ); - try self.truncateRegister(lhs_ty, tmp_reg); - try self.genSetMem( - .{ .frame = frame_addr.index }, - off, - tmp_ty, - .{ .register = tmp_reg }, - .{}, - ); - }, - else => {}, - } - const dst_lock = switch (dst_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - // shift right - const tmp_mcv = try self.genShiftBinOp(.shr, null, dst_mcv, rhs_mcv, lhs_ty, rhs_ty); - var tmp_temp = try self.tempInit(lhs_ty, tmp_mcv); - - // check if overflow happens - const cc_temp = lhs_temp1.cmpInts(.neq, &tmp_temp, self) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try lhs_temp1.die(self); - try tmp_temp.die(self); - const overflow_reloc = try self.genCondBrMir(lhs_ty, cc_temp.tracking(self).short); - try cc_temp.die(self); - - // if overflow, - // for unsigned integers, the saturating result is just its max - // for signed integers, - // if lhs is positive, the result is its max - // if lhs is negative, it is min - switch (lhs_ty.intInfo(zcu).signedness) { - .unsigned => { - const bound_mcv = try self.lowerValue(try lhs_ty.maxIntScalar(self.pt, lhs_ty)); - try self.genCopy(lhs_ty, dst_mcv, bound_mcv, .{}); - }, - .signed => { - // check the sign of lhs - // TODO: optimize this. - // we only need the highest bit so shifting the highest part of lhs_mcv - // is enough to check the signedness. other parts can be skipped here. - var lhs_temp2 = try self.tempInit(lhs_ty, lhs_mcv); - var zero_temp = try self.tempInit(lhs_ty, try self.lowerValue(try self.pt.intValue(lhs_ty, 0))); - const sign_cc_temp = lhs_temp2.cmpInts(.lt, &zero_temp, self) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try lhs_temp2.die(self); - try zero_temp.die(self); - const sign_reloc_condbr = try self.genCondBrMir(lhs_ty, sign_cc_temp.tracking(self).short); - try sign_cc_temp.die(self); - - // if it is negative - const min_mcv = try self.lowerValue(try lhs_ty.minIntScalar(self.pt, lhs_ty)); - try self.genCopy(lhs_ty, dst_mcv, min_mcv, .{}); - const sign_reloc_br = try self.asmJmpReloc(undefined); - self.performReloc(sign_reloc_condbr); - - // if it is positive - const max_mcv = try self.lowerValue(try lhs_ty.maxIntScalar(self.pt, lhs_ty)); - try self.genCopy(lhs_ty, dst_mcv, max_mcv, .{}); - self.performReloc(sign_reloc_br); - }, - } - - self.performReloc(overflow_reloc); - break :result dst_mcv; - }, - else => { - return self.fail("TODO implement shl_sat for {} op type {}", .{ self.target.cpu.arch, lhs_ty.zigTypeTag(zcu) }); - }, - } - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = result: { - const pl_ty = self.typeOfIndex(inst); - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const opt_mcv = try self.resolveInst(ty_op.operand); - if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) { - const pl_mcv: MCValue = switch (opt_mcv) { - .register_overflow => |ro| pl: { - self.eflags_inst = null; // actually stop tracking the overflow part - break :pl .{ .register = ro.reg }; - }, - else => opt_mcv, - }; - switch (pl_mcv) { - .register => |pl_reg| try self.truncateRegister(pl_ty, pl_reg), - else => {}, - } - break :result pl_mcv; - } - - const pl_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) { - else => opt_mcv, - .register_overflow => |ro| .{ .register = ro.reg }, - }, .{}); - break :result pl_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dst_ty = self.typeOfIndex(inst); - const opt_mcv = try self.resolveInst(ty_op.operand); - - const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) - opt_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv); - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = result: { - const dst_ty = self.typeOfIndex(inst); - const src_ty = self.typeOf(ty_op.operand); - const opt_ty = src_ty.childType(zcu); - const src_mcv = try self.resolveInst(ty_op.operand); - - if (opt_ty.optionalReprIsPayload(zcu)) { - break :result if (self.liveness.isUnused(inst)) - .unreach - else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); - } - - const dst_mcv: MCValue = if (src_mcv.isRegister() and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else if (self.liveness.isUnused(inst)) - .{ .register = try self.copyToTmpRegister(dst_ty, src_mcv) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); - - const pl_ty = dst_ty.childType(zcu); - const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); - try self.genSetMem( - .{ .reg = dst_mcv.getReg().? }, - pl_abi_size, - .bool, - .{ .immediate = 1 }, - .{}, - ); - break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const err_union_ty = self.typeOf(ty_op.operand); - const err_ty = err_union_ty.errorUnionSet(zcu); - const payload_ty = err_union_ty.errorUnionPayload(zcu); - const operand = try self.resolveInst(ty_op.operand); - - const result: MCValue = result: { - if (err_ty.errorSetIsEmpty(zcu)) { - break :result MCValue{ .immediate = 0 }; - } - - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - break :result try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); - } - - const err_off = codegen.errUnionErrorOffset(payload_ty, zcu); - switch (operand) { - .register => |reg| { - // TODO reuse operand - const eu_lock = self.register_manager.lockReg(reg); - defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); - - const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); - if (err_off > 0) try self.genShiftBinOpMir( - .{ ._r, .sh }, - err_union_ty, - result, - .u8, - .{ .immediate = @as(u6, @intCast(err_off * 8)) }, - ) else try self.truncateRegister(.anyerror, result.register); - break :result result; - }, - .load_frame => |frame_addr| break :result .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + @as(i32, @intCast(err_off)), - } }, - else => return self.fail("TODO implement unwrap_err_err for {f}", .{operand}), - } - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand_ty = self.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); - const result = try self.genUnwrapErrUnionPayloadMir(inst, operand_ty, operand); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -// *(E!T) -> E -fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const src_ty = self.typeOf(ty_op.operand); - const src_mcv = try self.resolveInst(ty_op.operand); - const src_reg = switch (src_mcv) { - .register => |reg| reg, - else => try self.copyToTmpRegister(src_ty, src_mcv), - }; - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - const eu_ty = src_ty.childType(zcu); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); - const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu)); - try self.asmRegisterMemory( - .{ ._, .mov }, - registerAlias(dst_reg, err_abi_size), - .{ - .base = .{ .reg = src_reg }, - .mod = .{ .rm = .{ - .size = .fromSize(err_abi_size), - .disp = err_off, - } }, - }, - ); - - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -// *(E!T) -> *T -fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand_ty = self.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); - const result = try self.genUnwrapErrUnionPayloadPtrMir(inst, operand_ty, operand); - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = result: { - const src_ty = self.typeOf(ty_op.operand); - const src_mcv = try self.resolveInst(ty_op.operand); - const src_reg = switch (src_mcv) { - .register => |reg| reg, - else => try self.copyToTmpRegister(src_ty, src_mcv), - }; - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - const eu_ty = src_ty.childType(zcu); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); - const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu)); - try self.asmMemoryImmediate( - .{ ._, .mov }, - .{ - .base = .{ .reg = src_reg }, - .mod = .{ .rm = .{ - .size = .fromSize(err_abi_size), - .disp = err_off, - } }, - }, - .u(0), - ); - - if (self.liveness.isUnused(inst)) break :result .unreach; - - const dst_ty = self.typeOfIndex(inst); - const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_reg - else - try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); - const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - try self.asmRegisterMemory( - .{ ._, .lea }, - registerAlias(dst_reg, dst_abi_size), - .{ - .base = .{ .reg = src_reg }, - .mod = .{ .rm = .{ .disp = pl_off } }, - }, - ); - break :result .{ .register = dst_reg }; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn genUnwrapErrUnionPayloadMir( - self: *CodeGen, - maybe_inst: ?Air.Inst.Index, - err_union_ty: Type, - err_union: MCValue, -) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const payload_ty = err_union_ty.errorUnionPayload(zcu); - - const result: MCValue = result: { - if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu)); - switch (err_union) { - .load_frame => |frame_addr| break :result .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + payload_off, - } }, - .register => |reg| { - // TODO reuse operand - const eu_lock = self.register_manager.lockReg(reg); - defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); - - const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp); - const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null) - try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union) - else - .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) }; - if (payload_off > 0) try self.genShiftBinOpMir( - .{ ._r, .sh }, - err_union_ty, - result_mcv, - .u8, - .{ .immediate = @as(u6, @intCast(payload_off * 8)) }, - ) else try self.truncateRegister(payload_ty, result_mcv.register); - break :result if (payload_in_gp) - result_mcv - else if (maybe_inst) |inst| - try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv) - else - .{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) }; - }, - else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {f}", .{err_union}), - } - }; - - return result; -} - -fn genUnwrapErrUnionPayloadPtrMir( - self: *CodeGen, - maybe_inst: ?Air.Inst.Index, - ptr_ty: Type, - ptr_mcv: MCValue, -) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const err_union_ty = ptr_ty.childType(zcu); - const payload_ty = err_union_ty.errorUnionPayload(zcu); - - const result: MCValue = result: { - const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu); - const result_mcv: MCValue = if (maybe_inst) |inst| - try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv) - else - .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }; - try self.genBinOpMir(.{ ._, .add }, ptr_ty, result_mcv, .{ .immediate = payload_off }); - break :result result_mcv; - }; - - return result; -} - -fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = result: { - const pl_ty = self.typeOf(ty_op.operand); - if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 }; - - const opt_ty = self.typeOfIndex(inst); - const pl_mcv = try self.resolveInst(ty_op.operand); - const same_repr = opt_ty.optionalReprIsPayload(zcu); - if (same_repr and self.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv; - - const pl_lock: ?RegisterLock = switch (pl_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (pl_lock) |lock| self.register_manager.unlockReg(lock); - - const opt_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(pl_ty, opt_mcv, pl_mcv, .{}); - - if (!same_repr) { - const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); - switch (opt_mcv) { - else => unreachable, - - .register => |opt_reg| { - try self.truncateRegister(pl_ty, opt_reg); - try self.asmRegisterImmediate( - .{ ._s, .bt }, - opt_reg, - .u(@as(u6, @intCast(pl_abi_size * 8))), - ); - }, - - .load_frame => |frame_addr| try self.asmMemoryImmediate( - .{ ._, .mov }, - .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .byte, - .disp = frame_addr.off + pl_abi_size, - } }, - }, - .u(1), - ), - } - } - break :result opt_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -/// T to E!T -fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const eu_ty = ty_op.ty.toType(); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - const operand = try self.resolveInst(ty_op.operand); - - const result: MCValue = result: { - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 }; - - const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu)); - const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); - const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{}); - try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{}); - break :result .{ .load_frame = .{ .index = frame_index } }; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -/// E to E!T -fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const eu_ty = ty_op.ty.toType(); - const pl_ty = eu_ty.errorUnionPayload(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - - const result: MCValue = result: { - if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand); - - const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu)); - const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); - const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{}); - const operand = try self.resolveInst(ty_op.operand); - try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{}); - break :result .{ .load_frame = .{ .index = frame_index } }; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = result: { - const src_mcv = try self.resolveInst(ty_op.operand); - const ptr_mcv: MCValue = switch (src_mcv) { - .register_pair => |regs| .{ .register = regs[0] }, - else => src_mcv, - }; - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { - switch (src_mcv) { - .register_pair => |regs| try self.freeValue(.{ .register = regs[1] }), - else => {}, - } - break :result ptr_mcv; - } - - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{}); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = result: { - const src_mcv = try self.resolveInst(ty_op.operand); - const len_mcv: MCValue = switch (src_mcv) { - .register_pair => |regs| .{ .register = regs[1] }, - .load_frame => |frame_addr| .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + 8, - } }, - else => return self.fail("TODO implement slice_len for {f}", .{src_mcv}), - }; - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { - switch (src_mcv) { - .register_pair => |regs| try self.freeValue(.{ .register = regs[0] }), - .load_frame => {}, - else => unreachable, - } - break :result len_mcv; - } - - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{}); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const src_ty = self.typeOf(ty_op.operand); - const src_mcv = try self.resolveInst(ty_op.operand); - const src_reg = switch (src_mcv) { - .register => |reg| reg, - else => try self.copyToTmpRegister(src_ty, src_mcv), - }; - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - const dst_ty = self.typeOfIndex(inst); - const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_reg - else - try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu)); - try self.asmRegisterMemory( - .{ ._, .lea }, - registerAlias(dst_reg, dst_abi_size), - .{ - .base = .{ .reg = src_reg }, - .mod = .{ .rm = .{ .disp = 8 } }, - }, - ); - - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dst_ty = self.typeOfIndex(inst); - const opt_mcv = try self.resolveInst(ty_op.operand); - - const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) - opt_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv); - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register { - const reg: Register = blk: { - switch (index) { - .immediate => |imm| { - // Optimisation: if index MCValue is an immediate, we can multiply in `comptime` - // and set the register directly to the scaled offset as an immediate. - const reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size }, .{}); - break :blk reg; - }, - else => { - const reg = try self.copyToTmpRegister(index_ty, index); - try self.genIntMulComplexOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size }); - break :blk reg; - }, - } - }; - return reg; -} - -fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const slice_ty = self.typeOf(lhs); - const slice_mcv = try self.resolveInst(lhs); - const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (slice_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - const elem_ty = slice_ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); - - const index_ty = self.typeOf(rhs); - const index_mcv = try self.resolveInst(rhs); - const index_mcv_lock: ?RegisterLock = switch (index_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size); - const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); - defer self.register_manager.unlockReg(offset_reg_lock); - - const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - try self.genSetReg(addr_reg, .usize, slice_mcv, .{}); - // TODO we could allocate register here, but need to expect addr register and potentially - // offset register. - try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{ - .register = offset_reg, - }); - return MCValue{ .register = addr_reg.to64() }; -} - -fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const result: MCValue = result: { - const elem_ty = self.typeOfIndex(inst); - if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const slice_ty = self.typeOf(bin_op.lhs); - const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); - const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs); - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.load(dst_mcv, slice_ptr_field_type, elem_ptr); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs); - return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); -} - -fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const result: MCValue = result: { - const array_ty = self.typeOf(bin_op.lhs); - const elem_ty = array_ty.childType(zcu); - - const array_mcv = try self.resolveInst(bin_op.lhs); - const array_lock: ?RegisterLock = switch (array_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (array_lock) |lock| self.register_manager.unlockReg(lock); - - const index_ty = self.typeOf(bin_op.rhs); - const index_mcv = try self.resolveInst(bin_op.rhs); - const index_lock = switch (index_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_lock) |lock| self.register_manager.unlockReg(lock); - - try self.spillEflagsIfOccupied(); - if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) { - const array_mat_mcv: MCValue = switch (array_mcv) { - else => array_mcv, - .register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) }, - }; - const array_mat_lock = switch (array_mat_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (array_mat_lock) |lock| self.register_manager.unlockReg(lock); - - switch (array_mat_mcv) { - .register => |array_reg| switch (array_reg.class()) { - .general_purpose => switch (index_mcv) { - .immediate => |index_imm| try self.asmRegisterImmediate( - .{ ._, .bt }, - array_reg.to64(), - .u(index_imm), - ), - else => try self.asmRegisterRegister( - .{ ._, .bt }, - array_reg.to64(), - switch (index_mcv) { - .register => |index_reg| index_reg, - else => try self.copyToTmpRegister(index_ty, index_mcv), - }.to64(), - ), - }, - .sse => { - const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{}); - switch (index_mcv) { - .immediate => |index_imm| try self.asmMemoryImmediate( - .{ ._, .bt }, - .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = @intCast(index_imm / 64 * 8), - } }, - }, - .u(index_imm % 64), - ), - else => try self.asmMemoryRegister( - .{ ._, .bt }, - .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .qword } }, - }, - switch (index_mcv) { - .register => |index_reg| index_reg, - else => try self.copyToTmpRegister(index_ty, index_mcv), - }.to64(), - ), - } - }, - else => unreachable, - }, - .load_frame => switch (index_mcv) { - .immediate => |index_imm| try self.asmMemoryImmediate( - .{ ._, .bt }, - try array_mat_mcv.mem(self, .{ - .size = .qword, - .disp = @intCast(index_imm / 64 * 8), - }), - .u(index_imm % 64), - ), - else => try self.asmMemoryRegister( - .{ ._, .bt }, - try array_mat_mcv.mem(self, .{ .size = .qword }), - switch (index_mcv) { - .register => |index_reg| index_reg, - else => try self.copyToTmpRegister(index_ty, index_mcv), - }.to64(), - ), - }, - .memory, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => switch (index_mcv) { - .immediate => |index_imm| try self.asmMemoryImmediate( - .{ ._, .bt }, - .{ - .base = .{ - .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()), - }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = @intCast(index_imm / 64 * 8), - } }, - }, - .u(index_imm % 64), - ), - else => try self.asmMemoryRegister( - .{ ._, .bt }, - .{ - .base = .{ - .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()), - }, - .mod = .{ .rm = .{ .size = .qword } }, - }, - switch (index_mcv) { - .register => |index_reg| index_reg, - else => try self.copyToTmpRegister(index_ty, index_mcv), - }.to64(), - ), - }, - else => return self.fail("TODO airArrayElemVal for {s} of {f}", .{ - @tagName(array_mat_mcv), array_ty.fmt(pt), - }), - } - - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - try self.asmSetccRegister(.c, dst_reg.to8()); - break :result .{ .register = dst_reg }; - } - - const elem_abi_size = elem_ty.abiSize(zcu); - const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_lock); - - switch (array_mcv) { - .register => { - const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{}); - try self.asmRegisterMemory( - .{ ._, .lea }, - addr_reg, - .{ .base = .{ .frame = frame_index } }, - ); - }, - .load_frame => |frame_addr| try self.asmRegisterMemory( - .{ ._, .lea }, - addr_reg, - .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ .disp = frame_addr.off } }, - }, - ), - .memory, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => try self.genSetReg(addr_reg, .usize, array_mcv.address(), .{}), - else => return self.fail("TODO airArrayElemVal_val for {s} of {f}", .{ - @tagName(array_mcv), array_ty.fmt(pt), - }), - } - - const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); - const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); - defer self.register_manager.unlockReg(offset_lock); - - // TODO we could allocate register here, but need to expect addr register and potentially - // offset register. - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genBinOpMir(.{ ._, .add }, .usize, .{ .register = addr_reg }, .{ .register = offset_reg }); - try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{}); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ptr_ty = self.typeOf(bin_op.lhs); - - // this is identical to the `airPtrElemPtr` codegen expect here an - // additional `mov` is needed at the end to get the actual value - - const result = result: { - const elem_ty = ptr_ty.elemType2(zcu); - if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - - const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); - const index_ty = self.typeOf(bin_op.rhs); - const index_mcv = try self.resolveInst(bin_op.rhs); - const index_lock = switch (index_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_lock) |lock| self.register_manager.unlockReg(lock); - - const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); - const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); - defer self.register_manager.unlockReg(offset_lock); - - const ptr_mcv = try self.resolveInst(bin_op.lhs); - const elem_ptr_reg = if (ptr_mcv.isRegister() and self.liveness.operandDies(inst, 0)) - ptr_mcv.register - else - try self.copyToTmpRegister(ptr_ty, ptr_mcv); - const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg); - defer self.register_manager.unlockReg(elem_ptr_lock); - try self.asmRegisterRegister( - .{ ._, .add }, - elem_ptr_reg, - offset_reg, - ); - - const dst_mcv = try self.allocRegOrMem(inst, true); - const dst_lock = switch (dst_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg }); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - - const result = result: { - const elem_ptr_ty = self.typeOfIndex(inst); - const base_ptr_ty = self.typeOf(extra.lhs); - - const base_ptr_mcv = try self.resolveInst(extra.lhs); - const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (base_ptr_lock) |lock| self.register_manager.unlockReg(lock); - - if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { - break :result if (self.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) - base_ptr_mcv - else - try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); - } - - const elem_ty = base_ptr_ty.elemType2(zcu); - const elem_abi_size = elem_ty.abiSize(zcu); - const index_ty = self.typeOf(extra.rhs); - const index_mcv = try self.resolveInst(extra.rhs); - const index_lock: ?RegisterLock = switch (index_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (index_lock) |lock| self.register_manager.unlockReg(lock); - - const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); - const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); - defer self.register_manager.unlockReg(offset_reg_lock); - - const dst_mcv = try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); - try self.genBinOpMir(.{ ._, .add }, elem_ptr_ty, dst_mcv, .{ .register = offset_reg }); - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); -} - -fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ptr_union_ty = self.typeOf(bin_op.lhs); - const union_ty = ptr_union_ty.childType(zcu); - const tag_ty = self.typeOf(bin_op.rhs); - const layout = union_ty.unionGetLayout(zcu); - - if (layout.tag_size == 0) { - return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const ptr = try self.resolveInst(bin_op.lhs); - const ptr_lock: ?RegisterLock = switch (ptr) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); - - const tag = try self.resolveInst(bin_op.rhs); - const tag_lock: ?RegisterLock = switch (tag) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (tag_lock) |lock| self.register_manager.unlockReg(lock); - - const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align.compare(.lt, layout.payload_align)) blk: { - // TODO reusing the operand - const reg = try self.copyToTmpRegister(ptr_union_ty, ptr); - try self.genBinOpMir( - .{ ._, .add }, - ptr_union_ty, - .{ .register = reg }, - .{ .immediate = layout.payload_size }, - ); - break :blk MCValue{ .register = reg }; - } else ptr; - - const ptr_tag_ty = try pt.adjustPtrTypeChild(ptr_union_ty, tag_ty); - try self.store(ptr_tag_ty, adjusted_ptr, tag, .{}); - - return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const tag_ty = self.typeOfIndex(inst); - const union_ty = self.typeOf(ty_op.operand); - const layout = union_ty.unionGetLayout(zcu); - - if (layout.tag_size == 0) { - return self.finishAir(inst, .none, .{ ty_op.operand, .none, .none }); - } - - // TODO reusing the operand - const operand = try self.resolveInst(ty_op.operand); - const operand_lock: ?RegisterLock = switch (operand) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); - - const tag_abi_size = tag_ty.abiSize(zcu); - const dst_mcv: MCValue = blk: { - switch (operand) { - .load_frame => |frame_addr| { - if (tag_abi_size <= 8) { - const off: i32 = @intCast(layout.tagOffset()); - break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{ - .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, - }); - } - - return self.fail( - "TODO implement get_union_tag for ABI larger than 8 bytes and operand {f}", - .{operand}, - ); - }, - .register => { - const shift: u6 = @intCast(layout.tagOffset() * 8); - const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand); - try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, result, .u8, .{ .immediate = shift }); - break :blk MCValue{ - .register = registerAlias(result.register, @intCast(layout.tag_size)), - }; - }, - else => return self.fail("TODO implement get_union_tag for {f}", .{operand}), - } - }; - - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = result: { - try self.spillEflagsIfOccupied(); - - const dst_ty = self.typeOfIndex(inst); - const src_ty = self.typeOf(ty_op.operand); - if (src_ty.zigTypeTag(zcu) == .vector) return self.fail("TODO implement airClz for {f}", .{ - src_ty.fmt(pt), - }); - - const src_mcv = try self.resolveInst(ty_op.operand); - const mat_src_mcv = switch (src_mcv) { - .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, - else => src_mcv, - }; - const mat_src_lock = switch (mat_src_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - const abi_size: u31 = @intCast(src_ty.abiSize(zcu)); - const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); - const has_lzcnt = self.hasFeature(.lzcnt); - if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) { - const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { - .load_frame => |src_frame_addr| src_frame_addr, - else => { - const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); - try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); - break :src_frame_addr .{ .index = src_frame_addr }; - }, - }; - - const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; - const extra_bits = abi_size * 8 - src_bits; - - const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); - defer self.register_manager.unlockReg(index_lock); - - try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len)); - switch (extra_bits) { - 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()), - else => try self.asmRegisterImmediate( - .{ ._, .mov }, - dst_reg.to32(), - .s(@as(i32, extra_bits) - 1), - ), - } - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32()); - const zero = try self.asmJccReloc(.z, undefined); - if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .de }, index_reg.to32()); - } - try self.asmMemoryImmediate(.{ ._, .cmp }, .{ - .base = .{ .frame = src_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = index_reg.to64(), - .scale = .@"8", - .disp = src_frame_addr.off, - } }, - }, .u(0)); - _ = try self.asmJccReloc(.e, loop); - try self.asmRegisterMemory(.{ ._r, .bs }, dst_reg.to64(), .{ - .base = .{ .frame = src_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = index_reg.to64(), - .scale = .@"8", - .disp = src_frame_addr.off, - } }, - }); - self.performReloc(zero); - try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6)); - try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32()); - try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1)); - try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32()); - break :result dst_mcv; - } - - if (has_lzcnt) { - if (src_bits <= 8) { - const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); - try self.truncateRegister(src_ty, wide_reg); - try self.genBinOpMir(.{ ._, .lzcnt }, .u32, dst_mcv, .{ .register = wide_reg }); - try self.genBinOpMir( - .{ ._, .sub }, - dst_ty, - dst_mcv, - .{ .immediate = 32 - src_bits }, - ); - } else if (src_bits <= 64) { - try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv); - const extra_bits = self.regExtraBits(src_ty); - if (extra_bits > 0) { - try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits }); - } - } else { - assert(src_bits <= 128); - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_mcv = MCValue{ .register = tmp_reg }; - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.genBinOpMir(.{ ._, .lzcnt }, .u64, dst_mcv, if (mat_src_mcv.isBase()) - mat_src_mcv - else - .{ .register = mat_src_mcv.register_pair[0] }); - try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 }); - try self.genBinOpMir(.{ ._, .lzcnt }, .u64, tmp_mcv, if (mat_src_mcv.isBase()) - mat_src_mcv.address().offset(8).deref() - else - .{ .register = mat_src_mcv.register_pair[1] }); - try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32()); - - if (src_bits < 128) try self.genBinOpMir( - .{ ._, .sub }, - dst_ty, - dst_mcv, - .{ .immediate = 128 - src_bits }, - ); - } - break :result dst_mcv; - } - - assert(src_bits <= 64); - const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2); - if (std.math.isPowerOfTwo(src_bits)) { - const imm_reg = try self.copyToTmpRegister(dst_ty, .{ - .immediate = src_bits ^ (src_bits - 1), - }); - const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg); - defer self.register_manager.unlockReg(imm_lock); - - if (src_bits <= 8) { - const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); - const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); - defer self.register_manager.unlockReg(wide_lock); - - try self.truncateRegister(src_ty, wide_reg); - try self.genBinOpMir(.{ ._r, .bs }, .u16, dst_mcv, .{ .register = wide_reg }); - } else try self.genBinOpMir(.{ ._r, .bs }, src_ty, dst_mcv, mat_src_mcv); - - try self.asmCmovccRegisterRegister( - .z, - registerAlias(dst_reg, cmov_abi_size), - registerAlias(imm_reg, cmov_abi_size), - ); - - try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 }); - } else { - const imm_reg = try self.copyToTmpRegister(dst_ty, .{ - .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)), - }); - const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg); - defer self.register_manager.unlockReg(imm_lock); - - const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); - const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); - defer self.register_manager.unlockReg(wide_lock); - - try self.truncateRegister(src_ty, wide_reg); - try self.genBinOpMir( - .{ ._r, .bs }, - if (src_bits <= 8) .u16 else src_ty, - dst_mcv, - .{ .register = wide_reg }, - ); - - try self.asmCmovccRegisterRegister( - .nz, - registerAlias(imm_reg, cmov_abi_size), - registerAlias(dst_reg, cmov_abi_size), - ); - - try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 }, .{}); - try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg }); - } - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = result: { - try self.spillEflagsIfOccupied(); - - const dst_ty = self.typeOfIndex(inst); - const src_ty = self.typeOf(ty_op.operand); - if (src_ty.zigTypeTag(zcu) == .vector) return self.fail("TODO implement airCtz for {f}", .{ - src_ty.fmt(pt), - }); - - const src_mcv = try self.resolveInst(ty_op.operand); - const mat_src_mcv = switch (src_mcv) { - .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, - else => src_mcv, - }; - const mat_src_lock = switch (mat_src_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const abi_size: u31 = @intCast(src_ty.abiSize(zcu)); - const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); - const has_bmi = self.hasFeature(.bmi); - if (src_bits > @as(u32, if (has_bmi) 128 else 64)) { - const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { - .load_frame => |src_frame_addr| src_frame_addr, - else => { - const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); - try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); - break :src_frame_addr .{ .index = src_frame_addr }; - }, - }; - - const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; - const extra_bits = abi_size * 8 - src_bits; - - const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); - defer self.register_manager.unlockReg(index_lock); - - try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1)); - switch (extra_bits) { - 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()), - 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()), - else => try self.asmRegisterImmediate( - .{ ._, .mov }, - dst_reg.to32(), - .s(-@as(i32, extra_bits)), - ), - } - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, index_reg.to32()); - } - try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len)); - const zero = try self.asmJccReloc(.nb, undefined); - try self.asmMemoryImmediate(.{ ._, .cmp }, .{ - .base = .{ .frame = src_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = index_reg.to64(), - .scale = .@"8", - .disp = src_frame_addr.off, - } }, - }, .u(0)); - _ = try self.asmJccReloc(.e, loop); - try self.asmRegisterMemory(.{ ._f, .bs }, dst_reg.to64(), .{ - .base = .{ .frame = src_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = index_reg.to64(), - .scale = .@"8", - .disp = src_frame_addr.off, - } }, - }); - self.performReloc(zero); - try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6)); - try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32()); - break :result dst_mcv; - } - - const wide_ty: Type = if (src_bits <= 8) .u16 else src_ty; - if (has_bmi) { - if (src_bits <= 64) { - const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0); - const masked_mcv = if (extra_bits > 0) masked: { - const tmp_mcv = tmp: { - if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) - break :tmp src_mcv; - try self.genSetReg(dst_reg, wide_ty, src_mcv, .{}); - break :tmp dst_mcv; - }; - try self.genBinOpMir( - .{ ._, .@"or" }, - wide_ty, - tmp_mcv, - .{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) << - @intCast(src_bits) }, - ); - break :masked tmp_mcv; - } else mat_src_mcv; - try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv); - } else { - assert(src_bits <= 128); - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_mcv = MCValue{ .register = tmp_reg }; - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) - mat_src_mcv - else - .{ .register = mat_src_mcv.register_pair[0] }; - const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) - mat_src_mcv.address().offset(8).deref() - else - .{ .register = mat_src_mcv.register_pair[1] }; - const masked_mcv = if (src_bits < 128) masked: { - try self.genCopy(.u64, dst_mcv, hi_mat_src_mcv, .{}); - try self.genBinOpMir( - .{ ._, .@"or" }, - .u64, - dst_mcv, - .{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) }, - ); - break :masked dst_mcv; - } else hi_mat_src_mcv; - try self.genBinOpMir(.{ ._, .tzcnt }, .u64, dst_mcv, masked_mcv); - try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 }); - try self.genBinOpMir(.{ ._, .tzcnt }, .u64, tmp_mcv, lo_mat_src_mcv); - try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32()); - } - break :result dst_mcv; - } - - assert(src_bits <= 64); - const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits }); - const width_lock = self.register_manager.lockRegAssumeUnused(width_reg); - defer self.register_manager.unlockReg(width_lock); - - if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) { - const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); - const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); - defer self.register_manager.unlockReg(wide_lock); - - try self.truncateRegister(src_ty, wide_reg); - try self.genBinOpMir(.{ ._f, .bs }, wide_ty, dst_mcv, .{ .register = wide_reg }); - } else try self.genBinOpMir(.{ ._f, .bs }, src_ty, dst_mcv, mat_src_mcv); - - const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2); - try self.asmCmovccRegisterRegister( - .z, - registerAlias(dst_reg, cmov_abi_size), - registerAlias(width_reg, cmov_abi_size), - ); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result: MCValue = result: { - try self.spillEflagsIfOccupied(); - - const src_ty = self.typeOf(ty_op.operand); - const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); - if (src_ty.zigTypeTag(zcu) == .vector or src_abi_size > 16) - return self.fail("TODO implement airPopCount for {f}", .{src_ty.fmt(pt)}); - const src_mcv = try self.resolveInst(ty_op.operand); - - const mat_src_mcv = switch (src_mcv) { - .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, - else => src_mcv, - }; - const mat_src_lock = switch (mat_src_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); - - if (src_abi_size <= 8) { - const dst_contains_src = - src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv); - const dst_reg = if (dst_contains_src) - src_mcv.getReg().? - else - try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genPopCount(dst_reg, src_ty, mat_src_mcv, dst_contains_src); - break :result .{ .register = dst_reg }; - } - - assert(src_abi_size > 8 and src_abi_size <= 16); - const tmp_regs = try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp); - const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); - defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); - - try self.genPopCount(tmp_regs[0], .usize, if (mat_src_mcv.isBase()) - mat_src_mcv - else - .{ .register = mat_src_mcv.register_pair[0] }, false); - const src_info = src_ty.intInfo(zcu); - const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1); - try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase()) - mat_src_mcv.address().offset(8).deref() - else - .{ .register = mat_src_mcv.register_pair[1] }, false); - try self.asmRegisterRegister(.{ ._, .add }, tmp_regs[0].to8(), tmp_regs[1].to8()); - break :result .{ .register = tmp_regs[0] }; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn genPopCount( - self: *CodeGen, - dst_reg: Register, - src_ty: Type, - src_mcv: MCValue, - dst_contains_src: bool, -) !void { - const pt = self.pt; - - const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu)); - if (self.hasFeature(.popcnt)) return self.genBinOpMir( - .{ ._, .popcnt }, - if (src_abi_size > 1) src_ty else .u32, - .{ .register = dst_reg }, - if (src_abi_size > 1) src_mcv else src: { - if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{}); - try self.truncateRegister(try src_ty.toUnsigned(pt), dst_reg); - break :src .{ .register = dst_reg }; - }, - ); - - const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8); - const imm_0_1: Immediate = .u(mask / 0b1_1); - const imm_00_11: Immediate = .u(mask / 0b01_01); - const imm_0000_1111: Immediate = .u(mask / 0b0001_0001); - const imm_0000_0001: Immediate = .u(mask / 0b1111_1111); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const dst = registerAlias(dst_reg, src_abi_size); - const tmp = registerAlias(tmp_reg, src_abi_size); - const imm = if (src_abi_size > 4) - try self.register_manager.allocReg(null, abi.RegisterClass.gp) - else - undefined; - - if (!dst_contains_src) try self.genSetReg(dst, src_ty, src_mcv, .{}); - // dst = operand - try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); - // tmp = operand - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1)); - // tmp = operand >> 1 - if (src_abi_size > 4) { - try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1); - try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); - } else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1); - // tmp = (operand >> 1) & 0x55...55 - try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp); - // dst = temp1 = operand - ((operand >> 1) & 0x55...55) - try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); - // tmp = temp1 - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2)); - // dst = temp1 >> 2 - if (src_abi_size > 4) { - try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11); - try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); - try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); - } else { - try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11); - try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11); - } - // tmp = temp1 & 0x33...33 - // dst = (temp1 >> 2) & 0x33...33 - try self.asmRegisterRegister(.{ ._, .add }, tmp, dst); - // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33) - try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp); - // dst = temp2 - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4)); - // tmp = temp2 >> 4 - try self.asmRegisterRegister(.{ ._, .add }, dst, tmp); - // dst = temp2 + (temp2 >> 4) - if (src_abi_size > 4) { - try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111); - try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001); - try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); - try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp); - } else { - try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111); - if (src_abi_size > 1) { - try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001); - } - } - // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f - // dst = temp3 * 0x01...01 - if (src_abi_size > 1) { - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8)); - } - // dst = (temp3 * 0x01...01) >> (bits - 8) -} - -fn genByteSwap( - self: *CodeGen, - inst: Air.Inst.Index, - src_ty: Type, - src_mcv: MCValue, - mem_ok: bool, -) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const has_movbe = self.hasFeature(.movbe); - - if (src_ty.zigTypeTag(zcu) == .vector) return self.fail( - "TODO implement genByteSwap for {f}", - .{src_ty.fmt(pt)}, - ); - - const src_lock = switch (src_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const abi_size: u32 = @intCast(src_ty.abiSize(zcu)); - switch (abi_size) { - 0 => unreachable, - 1 => return if ((mem_ok or src_mcv.isRegister()) and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv), - 2 => if ((mem_ok or src_mcv.isRegister()) and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - { - try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 }); - return src_mcv; - }, - 3...8 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { - try self.genUnOpMir(.{ .b_, .swap }, src_ty, src_mcv); - return src_mcv; - }, - 9...16 => { - const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) { - .register => { - const frame_index = try self.allocFrameIndex(.initSpill(src_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{}); - break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } }; - }, - .register_pair => |src_regs| if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { - for (src_regs) |src_reg| try self.asmRegister(.{ .b_, .swap }, src_reg.to64()); - return .{ .register_pair = .{ src_regs[1], src_regs[0] } }; - } else src_mcv, - else => src_mcv, - }; - - const dst_regs = - try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp); - const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); - defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); - - for (dst_regs, 0..) |dst_reg, limb_index| { - if (mat_src_mcv.isBase()) { - try self.asmRegisterMemory( - .{ if (has_movbe) ._be else ._, .mov }, - dst_reg.to64(), - try mat_src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }), - ); - if (!has_movbe) try self.asmRegister(.{ .b_, .swap }, dst_reg.to64()); - } else { - try self.asmRegisterRegister( - .{ ._, .mov }, - dst_reg.to64(), - mat_src_mcv.register_pair[limb_index].to64(), - ); - try self.asmRegister(.{ .b_, .swap }, dst_reg.to64()); - } - } - return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } }; - }, - else => { - const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; - - const temp_regs = - try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); - const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs); - defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32()); - try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1)); - - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterMemory( - .{ if (has_movbe) ._be else ._, .mov }, - temp_regs[2].to64(), - .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[0].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off, - } }, - }, - ); - try self.asmRegisterMemory( - .{ if (has_movbe) ._be else ._, .mov }, - temp_regs[3].to64(), - .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[1].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off, - } }, - }, - ); - if (!has_movbe) { - try self.asmRegister(.{ .b_, .swap }, temp_regs[2].to64()); - try self.asmRegister(.{ .b_, .swap }, temp_regs[3].to64()); - } - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[0].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off, - } }, - }, temp_regs[3].to64()); - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[1].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off, - } }, - }, temp_regs[2].to64()); - if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32()); - try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32()); - } - try self.asmRegisterRegister(.{ ._, .cmp }, temp_regs[0].to32(), temp_regs[1].to32()); - _ = try self.asmJccReloc(.be, loop); - return dst_mcv; - }, - } - - const dst_mcv: MCValue = if (mem_ok and has_movbe and src_mcv.isRegister()) - try self.allocRegOrMem(inst, true) - else - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) }; - if (dst_mcv.getReg()) |dst_reg| { - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register); - defer self.register_manager.unlockReg(dst_lock); - - try self.genSetReg(dst_reg, src_ty, src_mcv, .{}); - switch (abi_size) { - else => unreachable, - 2 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }), - 3...8 => try self.genUnOpMir(.{ .b_, .swap }, src_ty, dst_mcv), - } - } else try self.genBinOpMir(.{ ._be, .mov }, src_ty, dst_mcv, src_mcv); - return dst_mcv; -} - -fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const src_ty = self.typeOf(ty_op.operand); - const src_bits: u32 = @intCast(src_ty.bitSize(zcu)); - const src_mcv = try self.resolveInst(ty_op.operand); - - const dst_mcv = try self.genByteSwap(inst, src_ty, src_mcv, true); - try self.genShiftBinOpMir( - .{ ._r, switch (if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned) { - .signed => .sa, - .unsigned => .sh, - } }, - src_ty, - dst_mcv, - if (src_bits > 256) .u16 else .u8, - .{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits }, - ); - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const src_ty = self.typeOf(ty_op.operand); - const abi_size: u32 = @intCast(src_ty.abiSize(zcu)); - const bit_size: u32 = @intCast(src_ty.bitSize(zcu)); - const src_mcv = try self.resolveInst(ty_op.operand); - - const dst_mcv = try self.genByteSwap(inst, src_ty, src_mcv, false); - const dst_locks: [2]?RegisterLock = switch (dst_mcv) { - .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null }, - .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs), - else => unreachable, - }; - defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const limb_abi_size: u32 = @min(abi_size, 8); - const tmp = registerAlias(tmp_reg, limb_abi_size); - const imm = if (limb_abi_size > 4) - try self.register_manager.allocReg(null, abi.RegisterClass.gp) - else - undefined; - - const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8); - const imm_0000_1111: Immediate = .u(mask / 0b0001_0001); - const imm_00_11: Immediate = .u(mask / 0b01_01); - const imm_0_1: Immediate = .u(mask / 0b1_1); - - for (dst_mcv.getRegs()) |dst_reg| { - const dst = registerAlias(dst_reg, limb_abi_size); - - // dst = temp1 = bswap(operand) - try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); - // tmp = temp1 - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4)); - // dst = temp1 >> 4 - if (limb_abi_size > 4) { - try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111); - try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); - try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); - } else { - try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111); - try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111); - } - // tmp = temp1 & 0x0f...0f - // dst = (temp1 >> 4) & 0x0f...0f - try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4)); - // tmp = (temp1 & 0x0f...0f) << 4 - try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp); - // dst = temp2 = ((temp1 >> 4) & 0x0f...0f) | ((temp1 & 0x0f...0f) << 4) - try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); - // tmp = temp2 - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2)); - // dst = temp2 >> 2 - if (limb_abi_size > 4) { - try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11); - try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); - try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); - } else { - try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11); - try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11); - } - // tmp = temp2 & 0x33...33 - // dst = (temp2 >> 2) & 0x33...33 - try self.asmRegisterMemory( - .{ ._, .lea }, - if (limb_abi_size > 4) tmp.to64() else tmp.to32(), - .{ - .base = .{ .reg = dst.to64() }, - .mod = .{ .rm = .{ - .index = tmp.to64(), - .scale = .@"4", - } }, - }, - ); - // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2) - try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp); - // dst = temp3 - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1)); - // tmp = temp3 >> 1 - if (limb_abi_size > 4) { - try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1); - try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); - try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); - } else { - try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1); - try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1); - } - // dst = temp3 & 0x55...55 - // tmp = (temp3 >> 1) & 0x55...55 - try self.asmRegisterMemory( - .{ ._, .lea }, - if (limb_abi_size > 4) dst.to64() else dst.to32(), - .{ - .base = .{ .reg = tmp.to64() }, - .mod = .{ .rm = .{ - .index = dst.to64(), - .scale = .@"2", - } }, - }, - ); - // dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1) - } - - const extra_bits = abi_size * 8 - bit_size; - const signedness: std.builtin.Signedness = - if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned; - if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) { - .signed => .{ ._r, .sa }, - .unsigned => .{ ._r, .sh }, - }, src_ty, dst_mcv, .u8, .{ .immediate = extra_bits }); - - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn floatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag, operand: Air.Inst.Ref, ty: Type) !void { - const pt = self.pt; - const zcu = pt.zcu; - - const result = result: { - const scalar_bits = ty.scalarType(zcu).floatBits(self.target); - if (scalar_bits == 80) { - if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement floatSign for {f}", .{ - ty.fmt(pt), - }); - - const src_mcv = try self.resolveInst(operand); - const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv: MCValue = .{ .register = .st0 }; - if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv)) - try self.register_manager.getKnownReg(.st0, inst); - - try self.genCopy(ty, dst_mcv, src_mcv, .{}); - switch (tag) { - .neg => try self.asmOpOnly(.{ .f_, .chs }), - .abs => try self.asmOpOnly(.{ .f_, .abs }), - else => unreachable, - } - break :result dst_mcv; - } - - const abi_size: u32 = switch (ty.abiSize(zcu)) { - 1...16 => 16, - 17...32 => 32, - else => return self.fail("TODO implement floatSign for {f}", .{ - ty.fmt(pt), - }), - }; - - const src_mcv = try self.resolveInst(operand); - const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv: MCValue = if (src_mcv.isRegister() and - self.reuseOperand(inst, operand, 0, src_mcv)) - src_mcv - else if (self.hasFeature(.avx)) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const vec_ty = try pt.vectorType(.{ - .len = @divExact(abi_size * 8, scalar_bits), - .child = (try pt.intType(.signed, scalar_bits)).ip_index, - }); - - const sign_mcv = try self.lowerValue(switch (tag) { - .neg => try vec_ty.minInt(pt, vec_ty), - .abs => try vec_ty.maxInt(pt, vec_ty), - else => unreachable, - }); - const sign_mem: Memory = if (sign_mcv.isBase()) - try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) }) - else - .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, sign_mcv.address()) }, - .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, - }; - - if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory( - switch (scalar_bits) { - 16, 128 => if (abi_size <= 16 or self.hasFeature(.avx2)) switch (tag) { - .neg => .{ .vp_, .xor }, - .abs => .{ .vp_, .@"and" }, - else => unreachable, - } else switch (tag) { - .neg => .{ .v_ps, .xor }, - .abs => .{ .v_ps, .@"and" }, - else => unreachable, - }, - 32 => switch (tag) { - .neg => .{ .v_ps, .xor }, - .abs => .{ .v_ps, .@"and" }, - else => unreachable, - }, - 64 => switch (tag) { - .neg => .{ .v_pd, .xor }, - .abs => .{ .v_pd, .@"and" }, - else => unreachable, - }, - 80 => return self.fail("TODO implement floatSign for {f}", .{ty.fmt(pt)}), - else => unreachable, - }, - registerAlias(dst_reg, abi_size), - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv), abi_size), - sign_mem, - ) else try self.asmRegisterMemory( - switch (scalar_bits) { - 16, 128 => switch (tag) { - .neg => .{ .p_, .xor }, - .abs => .{ .p_, .@"and" }, - else => unreachable, - }, - 32 => switch (tag) { - .neg => .{ ._ps, .xor }, - .abs => .{ ._ps, .@"and" }, - else => unreachable, - }, - 64 => switch (tag) { - .neg => .{ ._pd, .xor }, - .abs => .{ ._pd, .@"and" }, - else => unreachable, - }, - 80 => return self.fail("TODO implement floatSign for {f}", .{ty.fmt(pt)}), - else => unreachable, - }, - registerAlias(dst_reg, abi_size), - sign_mem, - ); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ operand, .none, .none }); -} - -fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ty = self.typeOf(un_op); - return self.floatSign(inst, tag, un_op, ty); -} - -fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: bits.RoundMode) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ty = self.typeOf(un_op); - - const result = result: { - switch (try self.genRoundLibcall(ty, .{ .air_ref = un_op }, mode)) { - .none => {}, - else => |dst_mcv| break :result dst_mcv, - } - - const src_mcv = try self.resolveInst(un_op); - const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - try self.genRound(ty, dst_reg, src_mcv, mode); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag { - const pt = self.pt; - const zcu = pt.zcu; - return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(self.target)) { - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (ty.childType(zcu).floatBits(self.target)) { - 32 => switch (ty.vectorLen(zcu)) { - 1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round }, - 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round }, - 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null, - else => null, - }, - 64 => switch (ty.vectorLen(zcu)) { - 1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round }, - 2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round }, - 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => null, - }, - else => unreachable, - } else null; -} - -fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: bits.RoundMode) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - if (self.getRoundTag(ty)) |_| return .none; - - if (ty.zigTypeTag(zcu) != .float) - return self.fail("TODO implement genRound for {f}", .{ty.fmt(pt)}); - - var sym_buf: ["__trunc?".len]u8 = undefined; - return try self.genCall(.{ .extern_func = .{ - .return_type = ty.toIntern(), - .param_types = &.{ty.toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "{s}{s}{s}", .{ - floatLibcAbiPrefix(ty), - switch (mode.direction) { - .down => "floor", - .up => "ceil", - .zero => "trunc", - else => unreachable, - }, - floatLibcAbiSuffix(ty), - }) catch unreachable, - } }, &.{ty}, &.{src_mcv}, .{}); -} - -fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: bits.RoundMode) !void { - const pt = self.pt; - const mir_tag = self.getRoundTag(ty) orelse { - const result = try self.genRoundLibcall(ty, src_mcv, mode); - return self.genSetReg(dst_reg, ty, result, .{}); - }; - const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); - const dst_alias = registerAlias(dst_reg, abi_size); - switch (mir_tag[0]) { - .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - mir_tag, - dst_alias, - dst_alias, - try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - mode.imm(), - ) else try self.asmRegisterRegisterRegisterImmediate( - mir_tag, - dst_alias, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv), abi_size), - mode.imm(), - ), - else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( - mir_tag, - dst_alias, - try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - mode.imm(), - ) else try self.asmRegisterRegisterImmediate( - mir_tag, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv), abi_size), - mode.imm(), - ), - } -} - -fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const ty = self.typeOf(ty_op.operand); - - const result: MCValue = result: { - const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(zcu)) { - else => null, - .int => switch (ty.abiSize(zcu)) { - 0 => unreachable, - 1...8 => { - try self.spillEflagsIfOccupied(); - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); - - try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv); - - const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); - switch (src_mcv) { - .register => |val_reg| try self.asmCmovccRegisterRegister( - .l, - registerAlias(dst_mcv.register, cmov_abi_size), - registerAlias(val_reg, cmov_abi_size), - ), - .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( - .l, - registerAlias(dst_mcv.register, cmov_abi_size), - try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }), - ), - else => { - const val_reg = try self.copyToTmpRegister(ty, src_mcv); - try self.asmCmovccRegisterRegister( - .l, - registerAlias(dst_mcv.register, cmov_abi_size), - registerAlias(val_reg, cmov_abi_size), - ); - }, - } - break :result dst_mcv; - }, - 9...16 => { - try self.spillEflagsIfOccupied(); - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_mcv = if (src_mcv == .register_pair and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { - const dst_regs = try self.register_manager.allocRegs( - 2, - .{ inst, inst }, - abi.RegisterClass.gp, - ); - const dst_mcv: MCValue = .{ .register_pair = dst_regs }; - const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); - defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); - - try self.genCopy(ty, dst_mcv, src_mcv, .{}); - break :dst dst_mcv; - }; - const dst_regs = dst_mcv.register_pair; - const dst_locks = self.register_manager.lockRegs(2, dst_regs); - defer for (dst_locks) |dst_lock| if (dst_lock) |lock| - self.register_manager.unlockReg(lock); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); - try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63)); - try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg); - try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg); - try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg); - try self.asmRegisterRegister(.{ ._, .sbb }, dst_regs[1], tmp_reg); - - break :result dst_mcv; - }, - else => { - const abi_size: u31 = @intCast(ty.abiSize(zcu)); - const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable; - - const tmp_regs = - try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp); - const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs); - defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); - - try self.spillEflagsIfOccupied(); - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.allocRegOrMem(inst, false); - - try self.asmMemoryImmediate( - .{ ._, .cmp }, - try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }), - .u(0), - ); - const positive = try self.asmJccReloc(.ns, undefined); - - try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[0].to32(), tmp_regs[0].to32()); - try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[1].to8(), tmp_regs[1].to8()); - - const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32()); - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1)); - try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = tmp_regs[0].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off, - } }, - }); - try self.asmSetccRegister(.c, tmp_regs[1].to8()); - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = tmp_regs[0].to64(), - .scale = .@"8", - .disp = dst_mcv.load_frame.off, - } }, - }, tmp_regs[2].to64()); - - if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, tmp_regs[0].to32()); - } - try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len)); - _ = try self.asmJccReloc(.b, neg_loop); - - self.performReloc(positive); - break :result dst_mcv; - }, - }, - .float => return self.floatSign(inst, .abs, ty_op.operand, ty), - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - else => null, - .int => switch (ty.childType(zcu).intInfo(zcu).bits) { - else => null, - 8 => switch (ty.vectorLen(zcu)) { - else => null, - 1...16 => if (self.hasFeature(.avx)) - .{ .vp_b, .abs } - else if (self.hasFeature(.ssse3)) - .{ .p_b, .abs } - else - null, - 17...32 => if (self.hasFeature(.avx2)) .{ .vp_b, .abs } else null, - }, - 16 => switch (ty.vectorLen(zcu)) { - else => null, - 1...8 => if (self.hasFeature(.avx)) - .{ .vp_w, .abs } - else if (self.hasFeature(.ssse3)) - .{ .p_w, .abs } - else - null, - 9...16 => if (self.hasFeature(.avx2)) .{ .vp_w, .abs } else null, - }, - 32 => switch (ty.vectorLen(zcu)) { - else => null, - 1...4 => if (self.hasFeature(.avx)) - .{ .vp_d, .abs } - else if (self.hasFeature(.ssse3)) - .{ .p_d, .abs } - else - null, - 5...8 => if (self.hasFeature(.avx2)) .{ .vp_d, .abs } else null, - }, - }, - .float => return self.floatSign(inst, .abs, ty_op.operand, ty), - }, - }) orelse return self.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)}); - - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv.getReg().? - else - try self.register_manager.allocReg(inst, self.regSetForType(ty)); - const dst_alias = registerAlias(dst_reg, abi_size); - if (src_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_alias, - try src_mcv.mem(self, .{ .size = self.memSize(ty) }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv), abi_size), - ); - break :result .{ .register = dst_reg }; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ty = self.typeOf(un_op); - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - - const result: MCValue = result: { - switch (ty.zigTypeTag(zcu)) { - .float => { - const float_bits = ty.floatBits(self.target); - if (switch (float_bits) { - 16 => !self.hasFeature(.f16c), - 32, 64 => false, - 80, 128 => true, - else => unreachable, - }) { - var sym_buf: ["__sqrt?".len]u8 = undefined; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = ty.toIntern(), - .param_types = &.{ty.toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "{s}sqrt{s}", .{ - floatLibcAbiPrefix(ty), - floatLibcAbiSuffix(ty), - }) catch unreachable, - } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{}); - } - }, - else => {}, - } - - const src_mcv = try self.resolveInst(un_op); - const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); - const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(self.target)) { - 16 => { - assert(self.hasFeature(.f16c)); - const mat_src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv); - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128()); - try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - break :result dst_mcv; - }, - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt }, - else => unreachable, - }, - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (ty.childType(zcu).floatBits(self.target)) { - 16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen(zcu)) { - 1 => { - try self.asmRegisterRegister( - .{ .v_ps, .cvtph2 }, - dst_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv)).to128(), - ); - try self.asmRegisterRegisterRegister( - .{ .v_ss, .sqrt }, - dst_reg, - dst_reg, - dst_reg, - ); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - break :result dst_mcv; - }, - 2...8 => { - const wide_reg = registerAlias(dst_reg, abi_size * 2); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_ps, .cvtph2 }, - wide_reg, - try src_mcv.mem(self, .{ .size = .fromSize( - @intCast(@divExact(wide_reg.bitSize(), 16)), - ) }), - ) else try self.asmRegisterRegister( - .{ .v_ps, .cvtph2 }, - wide_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv)).to128(), - ); - try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - wide_reg, - bits.RoundMode.imm(.{}), - ); - break :result dst_mcv; - }, - else => null, - } else null, - 32 => switch (ty.vectorLen(zcu)) { - 1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt }, - 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt }, - 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null, - else => null, - }, - 64 => switch (ty.vectorLen(zcu)) { - 1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt }, - 2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt }, - 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null, - else => null, - }, - 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }) orelse return self.fail("TODO implement airSqrt for {f}", .{ty.fmt(pt)}); - switch (mir_tag[0]) { - .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( - mir_tag, - dst_reg, - dst_reg, - try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_reg, - dst_reg, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv), abi_size), - ), - else => if (src_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_reg, - try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_reg, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv), abi_size), - ), - } - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ty = self.typeOf(un_op); - var sym_buf: ["__round?".len]u8 = undefined; - const result = try self.genCall(.{ .extern_func = .{ - .return_type = ty.toIntern(), - .param_types = &.{ty.toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "{s}{s}{s}", .{ - floatLibcAbiPrefix(ty), - switch (tag) { - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .round, - => @tagName(tag), - else => unreachable, - }, - floatLibcAbiSuffix(ty), - }) catch unreachable, - } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{}); - return self.finishAir(inst, result, .{ un_op, .none, .none }); -} - -fn reuseOperand( - self: *CodeGen, - inst: Air.Inst.Index, - operand: Air.Inst.Ref, - op_index: Air.Liveness.OperandInt, - mcv: MCValue, -) bool { - return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); -} - -fn reuseOperandAdvanced( - self: *CodeGen, - inst: Air.Inst.Index, - operand: Air.Inst.Ref, - op_index: Air.Liveness.OperandInt, - mcv: MCValue, - maybe_tracked_inst: ?Air.Inst.Index, -) bool { - if (!self.liveness.operandDies(inst, op_index)) - return false; - - switch (mcv) { - .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| { - // If it's in the registers table, need to associate the register(s) with the - // new instruction. - if (maybe_tracked_inst) |tracked_inst| { - if (!self.register_manager.isRegFree(reg)) { - if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { - self.register_manager.registers[index] = tracked_inst; - } - } - } else self.register_manager.freeReg(reg); - }, - .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, - else => return false, - } - switch (mcv) { - .eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst, - else => {}, - } - - // Prevent the operand deaths processing code from deallocating it. - self.reused_operands.set(op_index); - const op_inst = operand.toIndex().?; - self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst); - - return true; -} - -fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const dst_ty = ptr_ty.childType(zcu); - if (!dst_ty.hasRuntimeBitsIgnoreComptime(zcu)) return; - switch (ptr_mcv) { - .none, - .unreach, - .dead, - .undef, - .eflags, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - => unreachable, // not a valid pointer - .immediate, - .register, - .register_offset, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref(), .{}), - .memory, - .indirect, - .load_frame, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => { - const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); - const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_lock); - - try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{}); - }, - .air_ref => |ptr_ref| try self.load(dst_mcv, ptr_ty, try self.resolveInst(ptr_ref)), - } -} - -fn store( - self: *CodeGen, - ptr_ty: Type, - ptr_mcv: MCValue, - src_mcv: MCValue, - opts: CopyOptions, -) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const src_ty = ptr_ty.childType(zcu); - if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return; - switch (ptr_mcv) { - .none, - .unreach, - .dead, - .undef, - .eflags, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - => unreachable, // not a valid pointer - .immediate, - .register, - .register_offset, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv, opts), - .memory, - .indirect, - .load_frame, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => { - const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); - const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_lock); - - try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv, opts); - }, - .air_ref => |ptr_ref| try self.store(ptr_ty, try self.resolveInst(ptr_ref), src_mcv, opts), - } -} - -fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const src_ty = self.typeOf(src_air); - if (src_ty.zigTypeTag(zcu) == .vector) - return self.fail("TODO implement genUnOp for {f}", .{src_ty.fmt(pt)}); - - var src_mcv = try self.resolveInst(src_air); - switch (src_mcv) { - .eflags => |cc| switch (tag) { - .not => { - if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) - return .{ .eflags = cc.negate() }; - try self.spillEflagsIfOccupied(); - src_mcv = try self.resolveInst(src_air); - }, - else => {}, - }, - else => {}, - } - - const src_lock = switch (src_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv: MCValue = dst: { - if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) break :dst src_mcv; - - const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, maybe_inst, true); - try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); - break :dst dst_mcv; - }; - const dst_lock = switch (dst_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const abi_size: u16 = @intCast(src_ty.abiSize(zcu)); - switch (tag) { - .not => { - const limb_abi_size: u16 = @min(abi_size, 8); - const int_info: InternPool.Key.IntType = if (src_ty.ip_index == .bool_type) - .{ .signedness = .unsigned, .bits = 1 } - else - src_ty.intInfo(zcu); - var byte_off: i32 = 0; - while (byte_off * 8 < int_info.bits) : (byte_off += limb_abi_size) { - const limb_bits: u16 = @intCast(@min(switch (int_info.signedness) { - .signed => abi_size * 8, - .unsigned => int_info.bits, - } - byte_off * 8, limb_abi_size * 8)); - const limb_ty = try pt.intType(int_info.signedness, limb_bits); - const limb_mcv = switch (byte_off) { - 0 => dst_mcv, - else => dst_mcv.address().offset(byte_off).deref(), - }; - - if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) { - const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits); - try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask }); - } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv); - } - }, - .neg => { - try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv); - const bit_size = src_ty.intInfo(zcu).bits; - if (abi_size * 8 > bit_size) { - if (dst_mcv.isRegister()) { - try self.truncateRegister(src_ty, dst_mcv.getReg().?); - } else { - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref(); - try self.genSetReg(tmp_reg, .usize, hi_mcv, .{}); - try self.truncateRegister(src_ty, tmp_reg); - try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{}); - } - } - }, - else => unreachable, - } - return dst_mcv; -} - -fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void { - const pt = self.pt; - const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu)); - if (abi_size > 8) return self.fail("TODO implement {} for {f}", .{ mir_tag, dst_ty.fmt(pt) }); - switch (dst_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .register_offset, - .eflags, - .register_overflow, - .register_mask, - .indirect_load_frame, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // unmodifiable destination - .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), - .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented - .memory, .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => { - const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_reg_lock); - - try self.genSetReg(addr_reg, .usize, dst_mcv.address(), .{}); - try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - } } }); - }, - .indirect, .load_frame => try self.asmMemory( - mir_tag, - try dst_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - ), - } -} - -/// Clobbers .rcx for non-immediate shift value. -fn genShiftBinOpMir( - self: *CodeGen, - tag: Mir.Inst.FixedTag, - lhs_ty: Type, - lhs_mcv: MCValue, - rhs_ty: Type, - rhs_mcv: MCValue, -) !void { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size: u31 = @intCast(lhs_ty.abiSize(zcu)); - const shift_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu)); - try self.spillEflagsIfOccupied(); - - if (abi_size > 16) { - const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; - assert(shift_abi_size >= 1 and shift_abi_size <= 2); - - const rcx_lock: ?RegisterLock = switch (rhs_mcv) { - .immediate => |shift_imm| switch (shift_imm) { - 0 => return, - else => null, - }, - else => lock: { - if (switch (rhs_mcv) { - .register => |rhs_reg| rhs_reg.id() != Register.rcx.id(), - else => true, - }) { - self.register_manager.getRegAssumeFree(.rcx, null); - try self.genSetReg(.rcx, rhs_ty, rhs_mcv, .{}); - } - break :lock self.register_manager.lockReg(.rcx); - }, - }; - defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); - - const temp_regs = try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); - const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs); - defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); - - switch (tag[0]) { - ._l => { - try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1)); - switch (rhs_mcv) { - .immediate => |shift_imm| try self.asmRegisterImmediate( - .{ ._, .mov }, - temp_regs[0].to32(), - .u(limbs_len - (shift_imm >> 6) - 1), - ), - else => { - try self.asmRegisterRegister( - .{ ._, .movzx }, - temp_regs[2].to32(), - registerAlias(.rcx, shift_abi_size), - ); - try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6))); - try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6)); - try self.asmRegisterRegister( - .{ ._, .mov }, - temp_regs[0].to32(), - temp_regs[1].to32(), - ); - try self.asmRegisterRegister( - .{ ._, .sub }, - temp_regs[0].to32(), - temp_regs[2].to32(), - ); - }, - } - }, - ._r => { - try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[1].to32(), temp_regs[1].to32()); - switch (rhs_mcv) { - .immediate => |shift_imm| try self.asmRegisterImmediate( - .{ ._, .mov }, - temp_regs[0].to32(), - .u(shift_imm >> 6), - ), - else => { - try self.asmRegisterRegister( - .{ ._, .movzx }, - temp_regs[0].to32(), - registerAlias(.rcx, shift_abi_size), - ); - try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6))); - try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6)); - }, - } - }, - else => unreachable, - } - - const slow_inc_dec = self.hasFeature(.slow_incdec); - if (switch (rhs_mcv) { - .immediate => |shift_imm| shift_imm >> 6 < limbs_len - 1, - else => true, - }) { - try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[2].to64(), .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[0].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off, - } }, - }); - const skip = switch (rhs_mcv) { - .immediate => undefined, - else => switch (tag[0]) { - ._l => try self.asmJccReloc(.z, undefined), - ._r => skip: { - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[0].to32(), - .u(limbs_len - 1), - ); - break :skip try self.asmJccReloc(.nb, undefined); - }, - else => unreachable, - }, - }; - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[3].to64(), .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[0].to64(), - .scale = .@"8", - .disp = switch (tag[0]) { - ._l => lhs_mcv.load_frame.off - 8, - ._r => lhs_mcv.load_frame.off + 8, - else => unreachable, - }, - } }, - }); - switch (rhs_mcv) { - .immediate => |shift_imm| try self.asmRegisterRegisterImmediate( - .{ switch (tag[0]) { - ._l => ._ld, - ._r => ._rd, - else => unreachable, - }, .sh }, - temp_regs[2].to64(), - temp_regs[3].to64(), - .u(shift_imm & std.math.maxInt(u6)), - ), - else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) { - ._l => ._ld, - ._r => ._rd, - else => unreachable, - }, .sh }, temp_regs[2].to64(), temp_regs[3].to64(), .cl), - } - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[1].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off, - } }, - }, temp_regs[2].to64()); - try self.asmRegisterRegister(.{ ._, .mov }, temp_regs[2].to64(), temp_regs[3].to64()); - switch (tag[0]) { - ._l => { - if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32()); - try self.asmRegister(.{ ._c, .de }, temp_regs[0].to32()); - } - _ = try self.asmJccReloc(.nz, loop); - }, - ._r => { - if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1)); - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, temp_regs[1].to32()); - try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32()); - } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[0].to32(), - .u(limbs_len - 1), - ); - _ = try self.asmJccReloc(.b, loop); - }, - else => unreachable, - } - switch (rhs_mcv) { - .immediate => {}, - else => self.performReloc(skip), - } - } else try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[2].to64(), .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = switch (tag[0]) { - ._l => lhs_mcv.load_frame.off, - ._r => lhs_mcv.load_frame.off + abi_size - 8, - else => unreachable, - }, - } }, - }); - switch (rhs_mcv) { - .immediate => |shift_imm| try self.asmRegisterImmediate( - tag, - temp_regs[2].to64(), - .u(shift_imm & std.math.maxInt(u6)), - ), - else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl), - } - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[1].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off, - } }, - }, temp_regs[2].to64()); - if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( - tag, - temp_regs[2].to64(), - .u(63), - ); - if (switch (rhs_mcv) { - .immediate => |shift_imm| shift_imm >> 6 > 0, - else => true, - }) { - const skip = switch (rhs_mcv) { - .immediate => undefined, - else => switch (tag[0]) { - ._l => skip: { - try self.asmRegisterRegister( - .{ ._, .@"test" }, - temp_regs[1].to32(), - temp_regs[1].to32(), - ); - break :skip try self.asmJccReloc(.z, undefined); - }, - ._r => skip: { - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[1].to32(), - .u(limbs_len - 1), - ); - break :skip try self.asmJccReloc(.nb, undefined); - }, - else => unreachable, - }, - }; - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - switch (tag[0]) { - ._l => if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32()); - }, - ._r => if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, temp_regs[1].to32()); - }, - else => unreachable, - } - if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[1].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off, - } }, - }, temp_regs[2].to64()) else try self.asmMemoryImmediate(.{ ._, .mov }, .{ - .base = .{ .frame = lhs_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .index = temp_regs[1].to64(), - .scale = .@"8", - .disp = lhs_mcv.load_frame.off, - } }, - }, .u(0)); - switch (tag[0]) { - ._l => _ = try self.asmJccReloc(.nz, loop), - ._r => { - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[1].to32(), - .u(limbs_len - 1), - ); - _ = try self.asmJccReloc(.b, loop); - }, - else => unreachable, - } - switch (rhs_mcv) { - .immediate => {}, - else => self.performReloc(skip), - } - } - return; - } - - assert(shift_abi_size == 1); - const shift_mcv: MCValue = shift: { - switch (rhs_mcv) { - .immediate => |shift_imm| switch (shift_imm) { - 0 => return, - else => break :shift rhs_mcv, - }, - .register => |rhs_reg| if (rhs_reg.id() == Register.rcx.id()) - break :shift rhs_mcv, - else => {}, - } - self.register_manager.getRegAssumeFree(.rcx, null); - try self.genSetReg(.cl, rhs_ty, rhs_mcv, .{}); - break :shift .{ .register = .rcx }; - }; - if (abi_size > 8) { - const info: struct { indices: [2]u31, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) { - ._l => .{ .indices = .{ 0, 1 }, .double_tag = .{ ._ld, .sh } }, - ._r => .{ .indices = .{ 1, 0 }, .double_tag = .{ ._rd, .sh } }, - else => unreachable, - }; - switch (lhs_mcv) { - .register_pair => |lhs_regs| switch (shift_mcv) { - .immediate => |shift_imm| if (shift_imm > 0 and shift_imm < 64) { - try self.asmRegisterRegisterImmediate( - info.double_tag, - lhs_regs[info.indices[1]], - lhs_regs[info.indices[0]], - .u(shift_imm), - ); - try self.asmRegisterImmediate( - tag, - lhs_regs[info.indices[0]], - .u(shift_imm), - ); - return; - } else { - assert(shift_imm < 128); - try self.asmRegisterRegister( - .{ ._, .mov }, - lhs_regs[info.indices[1]], - lhs_regs[info.indices[0]], - ); - if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( - tag, - lhs_regs[info.indices[0]], - .u(63), - ) else try self.asmRegisterRegister( - .{ ._, .xor }, - lhs_regs[info.indices[0]], - lhs_regs[info.indices[0]], - ); - if (shift_imm > 64) try self.asmRegisterImmediate( - tag, - lhs_regs[info.indices[1]], - .u(shift_imm - 64), - ); - return; - }, - .register => |shift_reg| { - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - if (tag[0] == ._r and tag[1] == .sa) { - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]); - try self.asmRegisterImmediate(tag, tmp_reg, .u(63)); - } else try self.asmRegisterRegister( - .{ ._, .xor }, - tmp_reg.to32(), - tmp_reg.to32(), - ); - try self.asmRegisterRegisterRegister( - info.double_tag, - lhs_regs[info.indices[1]], - lhs_regs[info.indices[0]], - shift_reg.to8(), - ); - try self.asmRegisterRegister( - tag, - lhs_regs[info.indices[0]], - shift_reg.to8(), - ); - try self.asmRegisterImmediate(.{ ._, .cmp }, shift_reg.to8(), .u(64)); - try self.asmCmovccRegisterRegister( - .ae, - lhs_regs[info.indices[1]], - lhs_regs[info.indices[0]], - ); - try self.asmCmovccRegisterRegister(.ae, lhs_regs[info.indices[0]], tmp_reg); - return; - }, - else => {}, - }, - .load_frame => |dst_frame_addr| { - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - switch (shift_mcv) { - .immediate => |shift_imm| if (shift_imm > 0 and shift_imm < 64) { - try self.asmRegisterMemory( - .{ ._, .mov }, - tmp_reg, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - ); - try self.asmMemoryRegisterImmediate( - info.double_tag, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[1] * 8, - } }, - }, - tmp_reg, - .u(shift_imm), - ); - try self.asmMemoryImmediate( - tag, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - .u(shift_imm), - ); - return; - } else { - assert(shift_imm < 128); - try self.asmRegisterMemory( - .{ ._, .mov }, - tmp_reg, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - ); - if (shift_imm > 64) try self.asmRegisterImmediate( - tag, - tmp_reg, - .u(shift_imm - 64), - ); - try self.asmMemoryRegister( - .{ ._, .mov }, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[1] * 8, - } }, - }, - tmp_reg, - ); - if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate( - tag, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - .u(63), - ) else { - try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32()); - try self.asmMemoryRegister( - .{ ._, .mov }, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - tmp_reg, - ); - } - return; - }, - .register => |shift_reg| { - const first_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const first_lock = self.register_manager.lockRegAssumeUnused(first_reg); - defer self.register_manager.unlockReg(first_lock); - - const second_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const second_lock = self.register_manager.lockRegAssumeUnused(second_reg); - defer self.register_manager.unlockReg(second_lock); - - try self.asmRegisterMemory( - .{ ._, .mov }, - first_reg, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - ); - try self.asmRegisterMemory( - .{ ._, .mov }, - second_reg, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[1] * 8, - } }, - }, - ); - if (tag[0] == ._r and tag[1] == .sa) { - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg); - try self.asmRegisterImmediate(tag, tmp_reg, .u(63)); - } else try self.asmRegisterRegister( - .{ ._, .xor }, - tmp_reg.to32(), - tmp_reg.to32(), - ); - try self.asmRegisterRegisterRegister( - info.double_tag, - second_reg, - first_reg, - shift_reg.to8(), - ); - try self.asmRegisterRegister(tag, first_reg, shift_reg.to8()); - try self.asmRegisterImmediate( - .{ ._, .cmp }, - shift_reg.to8(), - .u(64), - ); - try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg); - try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg); - try self.asmMemoryRegister( - .{ ._, .mov }, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[1] * 8, - } }, - }, - second_reg, - ); - try self.asmMemoryRegister( - .{ ._, .mov }, - .{ - .base = .{ .frame = dst_frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_frame_addr.off + info.indices[0] * 8, - } }, - }, - first_reg, - ); - return; - }, - else => {}, - } - }, - else => {}, - } - } else switch (lhs_mcv) { - .register => |lhs_reg| switch (shift_mcv) { - .immediate => |shift_imm| return self.asmRegisterImmediate( - tag, - registerAlias(lhs_reg, abi_size), - .u(shift_imm), - ), - .register => |shift_reg| return self.asmRegisterRegister( - tag, - registerAlias(lhs_reg, abi_size), - shift_reg.to8(), - ), - else => {}, - }, - .memory, .indirect, .load_frame => { - const lhs_mem: Memory = switch (lhs_mcv) { - .memory => |addr| .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse - return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ - @tagName(lhs_mcv), - @tagName(shift_mcv), - }), - } }, - }, - .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = reg_off.off, - } }, - }, - .load_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = frame_addr.off, - } }, - }, - else => unreachable, - }; - switch (shift_mcv) { - .immediate => |shift_imm| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)), - .register => |shift_reg| return self.asmMemoryRegister( - tag, - lhs_mem, - shift_reg.to8(), - ), - else => {}, - } - }, - else => {}, - } - return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ - @tagName(lhs_mcv), - @tagName(shift_mcv), - }); -} - -fn genBinOp( - self: *CodeGen, - maybe_inst: ?Air.Inst.Index, - air_tag: Air.Inst.Tag, - lhs_air: Air.Inst.Ref, - rhs_air: Air.Inst.Ref, -) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const lhs_ty = self.typeOf(lhs_air); - const rhs_ty = self.typeOf(rhs_air); - const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); - - if (lhs_ty.isRuntimeFloat()) libcall: { - const float_bits = lhs_ty.floatBits(self.target); - const type_needs_libcall = switch (float_bits) { - 16 => !self.hasFeature(.f16c), - 32, 64 => false, - 80, 128 => true, - else => unreachable, - }; - switch (air_tag) { - .rem, .mod => {}, - else => if (!type_needs_libcall) break :libcall, - } - var sym_buf: ["__mod?f3".len]u8 = undefined; - const sym = switch (air_tag) { - .add, - .sub, - .mul, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => std.fmt.bufPrint(&sym_buf, "__{s}{c}f3", .{ - @tagName(air_tag)[0..3], - floatCompilerRtAbiName(float_bits), - }), - .rem, .mod, .min, .max => std.fmt.bufPrint(&sym_buf, "{s}f{s}{s}", .{ - floatLibcAbiPrefix(lhs_ty), - switch (air_tag) { - .rem, .mod => "mod", - .min => "min", - .max => "max", - else => unreachable, - }, - floatLibcAbiSuffix(lhs_ty), - }), - else => return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - } catch unreachable; - const result = try self.genCall(.{ .extern_func = .{ - .return_type = lhs_ty.toIntern(), - .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() }, - .sym = sym, - } }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } }, .{}); - return switch (air_tag) { - .mod => result: { - const adjusted: MCValue = if (type_needs_libcall) adjusted: { - var add_sym_buf: ["__add?f3".len]u8 = undefined; - break :adjusted try self.genCall(.{ .extern_func = .{ - .return_type = lhs_ty.toIntern(), - .param_types = &.{ - lhs_ty.toIntern(), - rhs_ty.toIntern(), - }, - .sym = std.fmt.bufPrint(&add_sym_buf, "__add{c}f3", .{ - floatCompilerRtAbiName(float_bits), - }) catch unreachable, - } }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } }, .{}); - } else switch (float_bits) { - 16, 32, 64 => adjusted: { - const dst_reg = switch (result) { - .register => |reg| reg, - else => if (maybe_inst) |inst| - (try self.copyToRegisterWithInstTracking(inst, lhs_ty, result)).register - else - try self.copyToTmpRegister(lhs_ty, result), - }; - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(rhs_air); - const src_mcv: MCValue = if (float_bits == 16) src: { - assert(self.hasFeature(.f16c)); - const tmp_reg = (try self.register_manager.allocReg( - null, - abi.RegisterClass.sse, - )).to128(); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_w, .insr }, - dst_reg, - dst_reg, - try rhs_mcv.mem(self, .{ .size = .word }), - .u(1), - ) else try self.asmRegisterRegisterRegister( - .{ .vp_, .unpcklwd }, - dst_reg, - dst_reg, - (if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, rhs_mcv)).to128(), - ); - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); - try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); - break :src .{ .register = tmp_reg }; - } else rhs_mcv; - - if (self.hasFeature(.avx)) { - const mir_tag: Mir.Inst.FixedTag = switch (float_bits) { - 16, 32 => .{ .v_ss, .add }, - 64 => .{ .v_sd, .add }, - else => unreachable, - }; - if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( - mir_tag, - dst_reg, - dst_reg, - try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_reg, - dst_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - } else { - const mir_tag: Mir.Inst.FixedTag = switch (float_bits) { - 32 => .{ ._ss, .add }, - 64 => .{ ._sd, .add }, - else => unreachable, - }; - if (src_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_reg, - try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - } - - if (float_bits == 16) try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - break :adjusted .{ .register = dst_reg }; - }, - 80, 128 => return self.fail("TODO implement genBinOp for {s} of {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - else => unreachable, - }; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = lhs_ty.toIntern(), - .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() }, - .sym = sym, - } }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } }, .{}); - }, - .div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{ - .direction = switch (air_tag) { - .div_trunc => .zero, - .div_floor => .down, - else => unreachable, - }, - .precision = .inexact, - }), - else => result, - }; - } - - const sse_op = switch (lhs_ty.zigTypeTag(zcu)) { - else => false, - .float => true, - .vector => switch (lhs_ty.childType(zcu).toIntern()) { - .bool_type, .u1_type => false, - else => true, - }, - }; - if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and - lhs_ty.scalarType(zcu).floatBits(self.target) == 80) or - lhs_ty.abiSize(zcu) > self.vectorSize(.float))) - return self.fail("TODO implement genBinOp for {s} {f}", .{ @tagName(air_tag), lhs_ty.fmt(pt) }); - - const maybe_mask_reg = switch (air_tag) { - else => null, - .rem, .mod => unreachable, - .max, .min => if (lhs_ty.scalarType(zcu).isRuntimeFloat()) registerAlias( - if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: { - try self.register_manager.getKnownReg(.xmm0, null); - break :mask .xmm0; - } else try self.register_manager.allocReg(null, abi.RegisterClass.sse), - abi_size, - ) else null, - }; - const mask_lock = - if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null; - defer if (mask_lock) |lock| self.register_manager.unlockReg(lock); - - const ordered_air: [2]Air.Inst.Ref = if (lhs_ty.isVector(zcu) and - switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .bool => false, - .int => switch (air_tag) { - .cmp_lt, .cmp_gte => true, - else => false, - }, - .float => switch (air_tag) { - .cmp_gte, .cmp_gt => true, - else => false, - }, - else => unreachable, - }) .{ rhs_air, lhs_air } else .{ lhs_air, rhs_air }; - - if (lhs_ty.isAbiInt(zcu)) for (ordered_air) |op_air| { - switch (try self.resolveInst(op_air)) { - .register => |op_reg| switch (op_reg.class()) { - .sse => try self.register_manager.getReg(op_reg, null), - else => {}, - }, - else => {}, - } - }; - - const lhs_mcv = try self.resolveInst(ordered_air[0]); - var rhs_mcv = try self.resolveInst(ordered_air[1]); - switch (lhs_mcv) { - .immediate => |imm| switch (imm) { - 0 => switch (air_tag) { - .sub, .sub_wrap => return self.genUnOp(maybe_inst, .neg, ordered_air[1]), - else => {}, - }, - else => {}, - }, - else => {}, - } - - const is_commutative = switch (air_tag) { - .add, - .add_wrap, - .mul, - .bool_or, - .bit_or, - .bool_and, - .bit_and, - .xor, - .min, - .max, - .cmp_eq, - .cmp_neq, - => true, - - else => false, - }; - - const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) { - .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null }, - .register_pair => |lhs_regs| locks: { - const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs); - break :locks .{ locks[0], locks[1] }; - }, - else => @splat(null), - }; - defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) { - .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null }, - .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs), - else => @splat(null), - }; - defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - var flipped = false; - var copied_to_dst = true; - const dst_mcv: MCValue = dst: { - const tracked_inst = switch (air_tag) { - else => maybe_inst, - .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null, - }; - if (maybe_inst) |inst| { - if ((!sse_op or lhs_mcv.isRegister()) and - self.reuseOperandAdvanced(inst, ordered_air[0], 0, lhs_mcv, tracked_inst)) - break :dst lhs_mcv; - if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and - self.reuseOperandAdvanced(inst, ordered_air[1], 1, rhs_mcv, tracked_inst)) - { - flipped = true; - break :dst rhs_mcv; - } - } - const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true); - if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx)) - copied_to_dst = false - else - try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{}); - rhs_mcv = try self.resolveInst(ordered_air[1]); - break :dst dst_mcv; - }; - const dst_locks: [2]?RegisterLock = switch (dst_mcv) { - .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null }, - .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs), - else => @splat(null), - }; - defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const unmat_src_mcv = if (flipped) lhs_mcv else rhs_mcv; - const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg| - if (self.hasFeature(.avx) and unmat_src_mcv.isRegister() and maybe_inst != null and - self.liveness.operandDies(maybe_inst.?, if (flipped) 0 else 1)) unmat_src_mcv else src: { - try self.genSetReg(mask_reg, rhs_ty, unmat_src_mcv, .{}); - break :src .{ .register = mask_reg }; - } - else - unmat_src_mcv; - const src_locks: [2]?RegisterLock = switch (src_mcv) { - .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null }, - .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs), - else => @splat(null), - }; - defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock); - - if (!sse_op) { - switch (air_tag) { - .add, - .add_wrap, - => try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv), - - .sub, - .sub_wrap, - => try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv), - - .ptr_add, - .ptr_sub, - => { - const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv); - const tmp_mcv = MCValue{ .register = tmp_reg }; - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const elem_size = lhs_ty.elemType2(zcu).abiSize(zcu); - try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size }); - try self.genBinOpMir( - switch (air_tag) { - .ptr_add => .{ ._, .add }, - .ptr_sub => .{ ._, .sub }, - else => unreachable, - }, - lhs_ty, - dst_mcv, - tmp_mcv, - ); - }, - - .bool_or, - .bit_or, - => try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv), - - .bool_and, - .bit_and, - => try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv), - - .xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv), - - .min, - .max, - => { - const resolved_src_mcv = switch (src_mcv) { - else => src_mcv, - .air_ref => |src_ref| try self.resolveInst(src_ref), - }; - - if (abi_size > 8) { - const dst_regs = switch (dst_mcv) { - .register_pair => |dst_regs| dst_regs, - else => dst: { - const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); - const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); - defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock); - - try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{}); - break :dst dst_regs; - }, - }; - const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs); - defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock| - self.register_manager.unlockReg(lock); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - const signed = lhs_ty.isSignedInt(zcu); - const cc: Condition = switch (air_tag) { - .min => if (signed) .nl else .nb, - .max => if (signed) .nge else .nae, - else => unreachable, - }; - - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); - if (src_mcv.isBase()) { - try self.asmRegisterMemory( - .{ ._, .cmp }, - dst_regs[0], - try src_mcv.mem(self, .{ .size = .qword }), - ); - try self.asmRegisterMemory( - .{ ._, .sbb }, - tmp_reg, - try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ); - try self.asmCmovccRegisterMemory( - cc, - dst_regs[0], - try src_mcv.mem(self, .{ .size = .qword }), - ); - try self.asmCmovccRegisterMemory( - cc, - dst_regs[1], - try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), - ); - } else { - try self.asmRegisterRegister( - .{ ._, .cmp }, - dst_regs[0], - src_mcv.register_pair[0], - ); - try self.asmRegisterRegister( - .{ ._, .sbb }, - tmp_reg, - src_mcv.register_pair[1], - ); - try self.asmCmovccRegisterRegister(cc, dst_regs[0], src_mcv.register_pair[0]); - try self.asmCmovccRegisterRegister(cc, dst_regs[1], src_mcv.register_pair[1]); - } - try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs }, .{}); - } else { - const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) { - .immediate, - .eflags, - .register_offset, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => true, - .memory => |addr| std.math.cast(i32, @as(i64, @bitCast(addr))) == null, - else => false, - .register_pair, - .register_overflow, - => unreachable, - }) - .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) } - else - resolved_src_mcv; - const mat_mcv_lock = switch (mat_src_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv); - - const int_info = lhs_ty.intInfo(zcu); - const cc: Condition = switch (int_info.signedness) { - .unsigned => switch (air_tag) { - .min => .a, - .max => .b, - else => unreachable, - }, - .signed => switch (air_tag) { - .min => .g, - .max => .l, - else => unreachable, - }, - }; - - const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(zcu))), 2); - const tmp_reg = switch (dst_mcv) { - .register => |reg| reg, - else => try self.copyToTmpRegister(lhs_ty, dst_mcv), - }; - const tmp_lock = self.register_manager.lockReg(tmp_reg); - defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); - switch (mat_src_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .register_overflow, - .register_mask, - .indirect_load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .register => |src_reg| try self.asmCmovccRegisterRegister( - cc, - registerAlias(tmp_reg, cmov_abi_size), - registerAlias(src_reg, cmov_abi_size), - ), - .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( - cc, - registerAlias(tmp_reg, cmov_abi_size), - switch (mat_src_mcv) { - .memory => |addr| .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = .fromSize(cmov_abi_size), - .disp = @intCast(@as(i64, @bitCast(addr))), - } }, - }, - .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ .rm = .{ - .size = .fromSize(cmov_abi_size), - .disp = reg_off.off, - } }, - }, - .load_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .fromSize(cmov_abi_size), - .disp = frame_addr.off, - } }, - }, - else => unreachable, - }, - ), - } - try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg }, .{}); - } - }, - - .cmp_eq, .cmp_neq => { - assert(lhs_ty.isVector(zcu) and lhs_ty.childType(zcu).toIntern() == .bool_type); - try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv); - switch (air_tag) { - .cmp_eq => try self.genUnOpMir(.{ ._, .not }, lhs_ty, dst_mcv), - .cmp_neq => {}, - else => unreachable, - } - }, - - else => return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - } - return dst_mcv; - } - - const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size); - const mir_tag = @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { - else => unreachable, - .float => switch (lhs_ty.floatBits(self.target)) { - 16 => { - assert(self.hasFeature(.f16c)); - const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); - - const tmp_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_w, .insr }, - dst_reg, - lhs_reg, - try src_mcv.mem(self, .{ .size = .word }), - .u(1), - ) else try self.asmRegisterRegisterRegister( - .{ .vp_, .unpcklwd }, - dst_reg, - lhs_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); - try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); - try self.asmRegisterRegisterRegister( - switch (air_tag) { - .add => .{ .v_ss, .add }, - .sub => .{ .v_ss, .sub }, - .mul => .{ .v_ss, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, - .max => .{ .v_ss, .max }, - .min => .{ .v_ss, .min }, - else => unreachable, - }, - dst_reg, - dst_reg, - tmp_reg, - ); - switch (air_tag) { - .div_trunc, .div_floor => try self.asmRegisterRegisterRegisterImmediate( - .{ .v_ss, .round }, - dst_reg, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{ - .direction = switch (air_tag) { - .div_trunc => .zero, - .div_floor => .down, - else => unreachable, - }, - .precision = .inexact, - }), - ), - else => {}, - } - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - return dst_mcv; - }, - 32 => switch (air_tag) { - .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, - .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, - .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul }, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div }, - .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, - .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, - else => unreachable, - }, - 64 => switch (air_tag) { - .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, - .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, - .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul }, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div }, - .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, - .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, - else => unreachable, - }, - 80, 128 => null, - else => unreachable, - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - else => null, - .int => switch (lhs_ty.childType(zcu).intInfo(zcu).bits) { - 8 => switch (lhs_ty.vectorLen(zcu)) { - 1...16 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx)) .{ .vp_b, .add } else .{ .p_b, .add }, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx)) .{ .vp_b, .sub } else .{ .p_b, .sub }, - .bit_and => if (self.hasFeature(.avx)) - .{ .vp_, .@"and" } - else - .{ .p_, .@"and" }, - .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, - .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, - .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_b, .mins } - else if (self.hasFeature(.sse4_1)) - .{ .p_b, .mins } - else - null, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_b, .minu } - else if (self.hasFeature(.sse4_1)) - .{ .p_b, .minu } - else - null, - }, - .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_b, .maxs } - else if (self.hasFeature(.sse4_1)) - .{ .p_b, .maxs } - else - null, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_b, .maxu } - else if (self.hasFeature(.sse4_1)) - .{ .p_b, .maxu } - else - null, - }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_b, .cmpgt } - else - .{ .p_b, .cmpgt }, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_b, .cmpeq } else .{ .p_b, .cmpeq }, - else => null, - }, - 17...32 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_b, .add } else null, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_b, .sub } else null, - .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, - .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, - .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, - .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_b, .mins } else null, - .unsigned => if (self.hasFeature(.avx)) .{ .vp_b, .minu } else null, - }, - .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_b, .maxs } else null, - .unsigned => if (self.hasFeature(.avx2)) .{ .vp_b, .maxu } else null, - }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) .{ .vp_b, .cmpgt } else null, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_b, .cmpeq } else null, - else => null, - }, - else => null, - }, - 16 => switch (lhs_ty.vectorLen(zcu)) { - 1...8 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx)) .{ .vp_w, .add } else .{ .p_w, .add }, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx)) .{ .vp_w, .sub } else .{ .p_w, .sub }, - .mul, - .mul_wrap, - => if (self.hasFeature(.avx)) .{ .vp_w, .mull } else .{ .p_d, .mull }, - .bit_and => if (self.hasFeature(.avx)) - .{ .vp_, .@"and" } - else - .{ .p_, .@"and" }, - .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, - .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, - .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_w, .mins } - else - .{ .p_w, .mins }, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_w, .minu } - else - .{ .p_w, .minu }, - }, - .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_w, .maxs } - else - .{ .p_w, .maxs }, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_w, .maxu } - else - .{ .p_w, .maxu }, - }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_w, .cmpgt } - else - .{ .p_w, .cmpgt }, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_w, .cmpeq } else .{ .p_w, .cmpeq }, - else => null, - }, - 9...16 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_w, .add } else null, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_w, .sub } else null, - .mul, - .mul_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_w, .mull } else null, - .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, - .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, - .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, - .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .mins } else null, - .unsigned => if (self.hasFeature(.avx)) .{ .vp_w, .minu } else null, - }, - .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .maxs } else null, - .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .maxu } else null, - }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) .{ .vp_w, .cmpgt } else null, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_w, .cmpeq } else null, - else => null, - }, - else => null, - }, - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1...4 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx)) .{ .vp_d, .add } else .{ .p_d, .add }, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx)) .{ .vp_d, .sub } else .{ .p_d, .sub }, - .mul, - .mul_wrap, - => if (self.hasFeature(.avx)) - .{ .vp_d, .mull } - else if (self.hasFeature(.sse4_1)) - .{ .p_d, .mull } - else - null, - .bit_and => if (self.hasFeature(.avx)) - .{ .vp_, .@"and" } - else - .{ .p_, .@"and" }, - .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, - .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, - .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_d, .mins } - else if (self.hasFeature(.sse4_1)) - .{ .p_d, .mins } - else - null, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_d, .minu } - else if (self.hasFeature(.sse4_1)) - .{ .p_d, .minu } - else - null, - }, - .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_d, .maxs } - else if (self.hasFeature(.sse4_1)) - .{ .p_d, .maxs } - else - null, - .unsigned => if (self.hasFeature(.avx)) - .{ .vp_d, .maxu } - else if (self.hasFeature(.sse4_1)) - .{ .p_d, .maxu } - else - null, - }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_d, .cmpgt } - else - .{ .p_d, .cmpgt }, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else .{ .p_d, .cmpeq }, - else => null, - }, - 5...8 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_d, .add } else null, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_d, .sub } else null, - .mul, - .mul_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_d, .mull } else null, - .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, - .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, - .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, - .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .mins } else null, - .unsigned => if (self.hasFeature(.avx)) .{ .vp_d, .minu } else null, - }, - .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .maxs } else null, - .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .maxu } else null, - }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) .{ .vp_d, .cmpgt } else null, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else null, - else => null, - }, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1...2 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx)) .{ .vp_q, .add } else .{ .p_q, .add }, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx)) .{ .vp_q, .sub } else .{ .p_q, .sub }, - .bit_and => if (self.hasFeature(.avx)) - .{ .vp_, .@"and" } - else - .{ .p_, .@"and" }, - .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, - .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, - .cmp_lt, - .cmp_lte, - .cmp_gte, - .cmp_gt, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) - .{ .vp_q, .cmpgt } - else if (self.hasFeature(.sse4_2)) - .{ .p_q, .cmpgt } - else - null, - .unsigned => null, - }, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) - .{ .vp_q, .cmpeq } - else if (self.hasFeature(.sse4_1)) - .{ .p_q, .cmpeq } - else - null, - else => null, - }, - 3...4 => switch (air_tag) { - .add, - .add_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_q, .add } else null, - .sub, - .sub_wrap, - => if (self.hasFeature(.avx2)) .{ .vp_q, .sub } else null, - .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, - .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, - .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, - .cmp_eq, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else null, - .cmp_lt, - .cmp_lte, - .cmp_gt, - .cmp_gte, - => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { - .signed => if (self.hasFeature(.avx)) .{ .vp_d, .cmpgt } else null, - .unsigned => null, - }, - else => null, - }, - else => null, - }, - else => null, - }, - .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { - 16 => tag: { - assert(self.hasFeature(.f16c)); - const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); - switch (lhs_ty.vectorLen(zcu)) { - 1 => { - const tmp_reg = - (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_w, .insr }, - dst_reg, - lhs_reg, - try src_mcv.mem(self, .{ .size = .word }), - .u(1), - ) else try self.asmRegisterRegisterRegister( - .{ .vp_, .unpcklwd }, - dst_reg, - lhs_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); - try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); - try self.asmRegisterRegisterRegister( - switch (air_tag) { - .add => .{ .v_ss, .add }, - .sub => .{ .v_ss, .sub }, - .mul => .{ .v_ss, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, - .max => .{ .v_ss, .max }, - .min => .{ .v_ss, .max }, - else => unreachable, - }, - dst_reg, - dst_reg, - tmp_reg, - ); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - return dst_mcv; - }, - 2 => { - const tmp_reg = (try self.register_manager.allocReg( - null, - abi.RegisterClass.sse, - )).to128(); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_d, .insr }, - dst_reg, - lhs_reg, - try src_mcv.mem(self, .{ .size = .dword }), - .u(1), - ) else try self.asmRegisterRegisterRegister( - .{ .v_ps, .unpckl }, - dst_reg, - lhs_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); - try self.asmRegisterRegisterRegister( - .{ .v_ps, .movhl }, - tmp_reg, - dst_reg, - dst_reg, - ); - try self.asmRegisterRegisterRegister( - switch (air_tag) { - .add => .{ .v_ps, .add }, - .sub => .{ .v_ps, .sub }, - .mul => .{ .v_ps, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, - .max => .{ .v_ps, .max }, - .min => .{ .v_ps, .max }, - else => unreachable, - }, - dst_reg, - dst_reg, - tmp_reg, - ); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - return dst_mcv; - }, - 3...4 => { - const tmp_reg = (try self.register_manager.allocReg( - null, - abi.RegisterClass.sse, - )).to128(); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, lhs_reg); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_ps, .cvtph2 }, - tmp_reg, - try src_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ .v_ps, .cvtph2 }, - tmp_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - try self.asmRegisterRegisterRegister( - switch (air_tag) { - .add => .{ .v_ps, .add }, - .sub => .{ .v_ps, .sub }, - .mul => .{ .v_ps, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, - .max => .{ .v_ps, .max }, - .min => .{ .v_ps, .max }, - else => unreachable, - }, - dst_reg, - dst_reg, - tmp_reg, - ); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg, - bits.RoundMode.imm(.{}), - ); - return dst_mcv; - }, - 5...8 => { - const tmp_reg = (try self.register_manager.allocReg( - null, - abi.RegisterClass.sse, - )).to256(); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), lhs_reg); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_ps, .cvtph2 }, - tmp_reg, - try src_mcv.mem(self, .{ .size = .xword }), - ) else try self.asmRegisterRegister( - .{ .v_ps, .cvtph2 }, - tmp_reg, - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), - ); - try self.asmRegisterRegisterRegister( - switch (air_tag) { - .add => .{ .v_ps, .add }, - .sub => .{ .v_ps, .sub }, - .mul => .{ .v_ps, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, - .max => .{ .v_ps, .max }, - .min => .{ .v_ps, .max }, - else => unreachable, - }, - dst_reg.to256(), - dst_reg.to256(), - tmp_reg, - ); - try self.asmRegisterRegisterImmediate( - .{ .v_, .cvtps2ph }, - dst_reg, - dst_reg.to256(), - bits.RoundMode.imm(.{}), - ); - return dst_mcv; - }, - else => break :tag null, - } - }, - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1 => switch (air_tag) { - .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, - .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, - .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul }, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div }, - .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, - .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, - .cmp_lt, - .cmp_lte, - .cmp_eq, - .cmp_gte, - .cmp_gt, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp }, - else => unreachable, - }, - 2...4 => switch (air_tag) { - .add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add }, - .sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub }, - .mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul }, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div }, - .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max }, - .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min }, - .cmp_lt, - .cmp_lte, - .cmp_eq, - .cmp_gte, - .cmp_gt, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp }, - else => unreachable, - }, - 5...8 => if (self.hasFeature(.avx)) switch (air_tag) { - .add => .{ .v_ps, .add }, - .sub => .{ .v_ps, .sub }, - .mul => .{ .v_ps, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, - .max => .{ .v_ps, .max }, - .min => .{ .v_ps, .min }, - .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp }, - else => unreachable, - } else null, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1 => switch (air_tag) { - .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, - .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, - .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul }, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div }, - .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, - .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, - .cmp_lt, - .cmp_lte, - .cmp_eq, - .cmp_gte, - .cmp_gt, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp }, - else => unreachable, - }, - 2 => switch (air_tag) { - .add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add }, - .sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub }, - .mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul }, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div }, - .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max }, - .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min }, - .cmp_lt, - .cmp_lte, - .cmp_eq, - .cmp_gte, - .cmp_gt, - .cmp_neq, - => if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp }, - else => unreachable, - }, - 3...4 => if (self.hasFeature(.avx)) switch (air_tag) { - .add => .{ .v_pd, .add }, - .sub => .{ .v_pd, .sub }, - .mul => .{ .v_pd, .mul }, - .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div }, - .max => .{ .v_pd, .max }, - .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp }, - .min => .{ .v_pd, .min }, - else => unreachable, - } else null, - else => null, - }, - 80, 128 => null, - else => unreachable, - }, - }, - }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }); - - const lhs_copy_reg = if (maybe_mask_reg) |_| registerAlias( - if (copied_to_dst) try self.copyToTmpRegister(lhs_ty, dst_mcv) else lhs_mcv.getReg().?, - abi_size, - ) else null; - const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null; - defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock); - - switch (mir_tag[1]) { - else => if (self.hasFeature(.avx)) { - const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); - if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( - mir_tag, - dst_reg, - lhs_reg, - try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { - else => .fromSize(abi_size), - .vector => dst_reg.size(), - } }), - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_reg, - lhs_reg, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), - ); - } else { - assert(copied_to_dst); - if (src_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_reg, - try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { - else => .fromSize(abi_size), - .vector => dst_reg.size(), - } }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_reg, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), - ); - }, - .cmp => { - const imm: Immediate = .u(switch (air_tag) { - .cmp_eq => 0, - .cmp_lt, .cmp_gt => 1, - .cmp_lte, .cmp_gte => 2, - .cmp_neq => 4, - else => unreachable, - }); - if (self.hasFeature(.avx)) { - const lhs_reg = - if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); - if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - mir_tag, - dst_reg, - lhs_reg, - try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { - else => .fromSize(abi_size), - .vector => dst_reg.size(), - } }), - imm, - ) else try self.asmRegisterRegisterRegisterImmediate( - mir_tag, - dst_reg, - lhs_reg, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), - imm, - ); - } else { - assert(copied_to_dst); - if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( - mir_tag, - dst_reg, - try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { - else => .fromSize(abi_size), - .vector => dst_reg.size(), - } }), - imm, - ) else try self.asmRegisterRegisterImmediate( - mir_tag, - dst_reg, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), - imm, - ); - } - }, - } - - switch (air_tag) { - .bit_and, .bit_or, .xor => {}, - .max, .min => if (maybe_mask_reg) |mask_reg| if (self.hasFeature(.avx)) { - const rhs_copy_reg = registerAlias(src_mcv.getReg().?, abi_size); - - try self.asmRegisterRegisterRegisterImmediate( - @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { - .float => switch (lhs_ty.floatBits(self.target)) { - 32 => .{ .v_ss, .cmp }, - 64 => .{ .v_sd, .cmp }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1 => .{ .v_ss, .cmp }, - 2...8 => .{ .v_ps, .cmp }, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1 => .{ .v_sd, .cmp }, - 2...4 => .{ .v_pd, .cmp }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - mask_reg, - rhs_copy_reg, - rhs_copy_reg, - bits.VexFloatPredicate.imm(.unord), - ); - try self.asmRegisterRegisterRegisterRegister( - @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { - .float => switch (lhs_ty.floatBits(self.target)) { - 32 => .{ .v_ps, .blendv }, - 64 => .{ .v_pd, .blendv }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1...8 => .{ .v_ps, .blendv }, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1...4 => .{ .v_pd, .blendv }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - dst_reg, - dst_reg, - lhs_copy_reg.?, - mask_reg, - ); - } else { - const has_blend = self.hasFeature(.sse4_1); - try self.asmRegisterRegisterImmediate( - @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { - .float => switch (lhs_ty.floatBits(self.target)) { - 32 => .{ ._ss, .cmp }, - 64 => .{ ._sd, .cmp }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1 => .{ ._ss, .cmp }, - 2...4 => .{ ._ps, .cmp }, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1 => .{ ._sd, .cmp }, - 2 => .{ ._pd, .cmp }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - mask_reg, - mask_reg, - bits.SseFloatPredicate.imm(if (has_blend) .unord else .ord), - ); - if (has_blend) try self.asmRegisterRegisterRegister( - @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { - .float => switch (lhs_ty.floatBits(self.target)) { - 32 => .{ ._ps, .blendv }, - 64 => .{ ._pd, .blendv }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1...4 => .{ ._ps, .blendv }, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1...2 => .{ ._pd, .blendv }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }), - dst_reg, - lhs_copy_reg.?, - mask_reg, - ) else { - const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(zcu)) { - .float => switch (lhs_ty.floatBits(self.target)) { - 32 => ._ps, - 64 => ._pd, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { - 32 => switch (lhs_ty.vectorLen(zcu)) { - 1...4 => ._ps, - else => null, - }, - 64 => switch (lhs_ty.vectorLen(zcu)) { - 1...2 => ._pd, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ - @tagName(air_tag), lhs_ty.fmt(pt), - }); - try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg); - try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?); - try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg); - } - }, - .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => { - switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { - .int => switch (air_tag) { - .cmp_lt, - .cmp_eq, - .cmp_gt, - => {}, - .cmp_lte, - .cmp_gte, - .cmp_neq, - => { - const unsigned_ty = try lhs_ty.toUnsigned(pt); - const not_mcv = try self.lowerValue(try unsigned_ty.maxInt(pt, unsigned_ty)); - const not_mem: Memory = if (not_mcv.isBase()) - try not_mcv.mem(self, .{ .size = .fromSize(abi_size) }) - else - .{ .base = .{ - .reg = try self.copyToTmpRegister(.usize, not_mcv.address()), - }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } }; - switch (mir_tag[0]) { - .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory( - .{ .vp_, .xor }, - dst_reg, - dst_reg, - not_mem, - ), - .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory( - .{ .p_, .xor }, - dst_reg, - not_mem, - ), - else => unreachable, - } - }, - else => unreachable, - }, - .float => {}, - else => unreachable, - } - - const gp_reg = try self.register_manager.allocReg(maybe_inst, abi.RegisterClass.gp); - const gp_lock = self.register_manager.lockRegAssumeUnused(gp_reg); - defer self.register_manager.unlockReg(gp_lock); - - try self.asmRegisterRegister(switch (mir_tag[0]) { - ._pd, ._sd, .p_q => .{ ._pd, .movmsk }, - ._ps, ._ss, .p_d => .{ ._ps, .movmsk }, - .p_b => .{ .p_b, .movmsk }, - .p_w => movmsk: { - try self.asmRegisterRegister(.{ .p_b, .ackssw }, dst_reg, dst_reg); - break :movmsk .{ .p_b, .movmsk }; - }, - .v_pd, .v_sd, .vp_q => .{ .v_pd, .movmsk }, - .v_ps, .v_ss, .vp_d => .{ .v_ps, .movmsk }, - .vp_b => .{ .vp_b, .movmsk }, - .vp_w => movmsk: { - try self.asmRegisterRegisterRegister( - .{ .vp_b, .ackssw }, - dst_reg, - dst_reg, - dst_reg, - ); - break :movmsk .{ .vp_b, .movmsk }; - }, - else => unreachable, - }, gp_reg.to32(), dst_reg); - return .{ .register = gp_reg }; - }, - else => unreachable, - } - - return dst_mcv; -} - -fn genBinOpMir( - self: *CodeGen, - mir_tag: Mir.Inst.FixedTag, - ty: Type, - dst_mcv: MCValue, - src_mcv: MCValue, -) !void { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - try self.spillEflagsIfOccupied(); - switch (dst_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register_overflow, - .register_mask, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // unmodifiable destination - .register, .register_pair, .register_triple, .register_quadruple, .register_offset => { - switch (dst_mcv) { - .register, .register_pair, .register_triple, .register_quadruple => {}, - .register_offset => |ro| assert(ro.off == 0), - else => unreachable, - } - for (dst_mcv.getRegs(), 0..) |dst_reg, dst_reg_i| { - const dst_reg_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); - - const mir_limb_tag: Mir.Inst.FixedTag = switch (dst_reg_i) { - 0 => mir_tag, - 1 => switch (mir_tag[1]) { - .add => .{ ._, .adc }, - .sub, .cmp => .{ ._, .sbb }, - .@"or", .@"and", .xor => mir_tag, - else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{ - @tagName(mir_tag[1]), - }), - }, - else => unreachable, - }; - const off: u4 = @intCast(dst_reg_i * 8); - const limb_abi_size = @min(abi_size - off, 8); - const dst_alias = registerAlias(dst_reg, limb_abi_size); - switch (src_mcv) { - .none, - .unreach, - .dead, - .undef, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - => unreachable, - .register, - .register_pair, - .register_triple, - .register_quadruple, - => try self.asmRegisterRegister( - mir_limb_tag, - dst_alias, - registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size), - ), - .immediate => |imm| { - assert(off == 0); - switch (self.regBitSize(ty)) { - 8 => try self.asmRegisterImmediate( - mir_limb_tag, - dst_alias, - if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small| - .s(small) - else - .u(@as(u8, @intCast(imm))), - ), - 16 => try self.asmRegisterImmediate( - mir_limb_tag, - dst_alias, - if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small| - .s(small) - else - .u(@as(u16, @intCast(imm))), - ), - 32 => try self.asmRegisterImmediate( - mir_limb_tag, - dst_alias, - if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| - .s(small) - else - .u(@as(u32, @intCast(imm))), - ), - 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| - try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small)) - else - try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias( - try self.copyToTmpRegister(ty, src_mcv), - limb_abi_size, - )), - else => unreachable, - } - }, - .eflags, - .register_offset, - .memory, - .indirect, - .load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => { - direct: { - try self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) { - .memory => |addr| .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = .fromSize(limb_abi_size), - .disp = std.math.cast(i32, addr + off) orelse break :direct, - } }, - }, - .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ .rm = .{ - .size = .fromSize(limb_abi_size), - .disp = reg_off.off + off, - } }, - }, - .load_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .fromSize(limb_abi_size), - .disp = frame_addr.off + off, - } }, - }, - else => break :direct, - }); - continue; - } - - switch (src_mcv) { - .eflags, - .register_offset, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => { - assert(off == 0); - const reg = try self.copyToTmpRegister(ty, src_mcv); - return self.genBinOpMir( - mir_limb_tag, - ty, - dst_mcv, - .{ .register = reg }, - ); - }, - .memory, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => { - const ptr_ty = try pt.singleConstPtrType(ty); - const addr_reg = try self.copyToTmpRegister(ptr_ty, src_mcv.address()); - return self.genBinOpMir(mir_limb_tag, ty, dst_mcv, .{ - .indirect = .{ .reg = addr_reg, .off = off }, - }); - }, - else => unreachable, - } - }, - .air_ref => |src_ref| return self.genBinOpMir( - mir_tag, - ty, - dst_mcv, - try self.resolveInst(src_ref), - ), - } - } - }, - .memory, - .indirect, - .load_frame, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => { - const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock }; - const limb_abi_size: u32 = @min(abi_size, 8); - - const dst_info: OpInfo = switch (dst_mcv) { - else => unreachable, - .memory, .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => dst: { - const dst_addr_reg = - (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); - const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); - errdefer self.register_manager.unlockReg(dst_addr_lock); - - try self.genSetReg(dst_addr_reg, .usize, dst_mcv.address(), .{}); - break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock }; - }, - .load_frame => null, - }; - defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock); - - const resolved_src_mcv = switch (src_mcv) { - else => src_mcv, - .air_ref => |src_ref| try self.resolveInst(src_ref), - }; - const src_info: OpInfo = switch (resolved_src_mcv) { - .none, - .unreach, - .dead, - .undef, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .immediate, - .eflags, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .indirect, - .load_frame, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => null, - .memory, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => src: { - switch (resolved_src_mcv) { - .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and - std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null) - break :src null, - .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, - else => unreachable, - } - - const src_addr_reg = - (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); - const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); - errdefer self.register_manager.unlockReg(src_addr_lock); - - try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{}); - break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; - }, - }; - defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); - - const ty_signedness = - if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned; - const limb_ty: Type = if (abi_size <= 8) ty else switch (ty_signedness) { - .signed => .usize, - .unsigned => .isize, - }; - var limb_i: usize = 0; - var off: i32 = 0; - while (off < abi_size) : ({ - limb_i += 1; - off += 8; - }) { - const mir_limb_tag: Mir.Inst.FixedTag = switch (limb_i) { - 0 => mir_tag, - else => switch (mir_tag[1]) { - .add => .{ ._, .adc }, - .sub, .cmp => .{ ._, .sbb }, - .@"or", .@"and", .xor => mir_tag, - else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{ - @tagName(mir_tag[1]), - }), - }, - }; - const dst_limb_mem: Memory = switch (dst_mcv) { - .memory, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => .{ - .base = .{ .reg = dst_info.?.addr_reg }, - .mod = .{ .rm = .{ - .size = .fromSize(limb_abi_size), - .disp = off, - } }, - }, - .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ .rm = .{ - .size = .fromSize(limb_abi_size), - .disp = reg_off.off + off, - } }, - }, - .load_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .fromSize(limb_abi_size), - .disp = frame_addr.off + off, - } }, - }, - else => unreachable, - }; - switch (resolved_src_mcv) { - .none, - .unreach, - .dead, - .undef, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .immediate => |src_imm| { - const imm: u64 = switch (limb_i) { - 0 => src_imm, - else => switch (ty_signedness) { - .signed => @bitCast(@as(i64, @bitCast(src_imm)) >> 63), - .unsigned => 0, - }, - }; - switch (self.regBitSize(limb_ty)) { - 8 => try self.asmMemoryImmediate( - mir_limb_tag, - dst_limb_mem, - if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small| - .s(small) - else - .u(@as(u8, @intCast(imm))), - ), - 16 => try self.asmMemoryImmediate( - mir_limb_tag, - dst_limb_mem, - if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small| - .s(small) - else - .u(@as(u16, @intCast(imm))), - ), - 32 => try self.asmMemoryImmediate( - mir_limb_tag, - dst_limb_mem, - if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| - .s(small) - else - .u(@as(u32, @intCast(imm))), - ), - 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| - try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small)) - else - try self.asmMemoryRegister( - mir_limb_tag, - dst_limb_mem, - registerAlias( - try self.copyToTmpRegister(limb_ty, .{ .immediate = imm }), - limb_abi_size, - ), - ), - else => unreachable, - } - }, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .eflags, - .memory, - .indirect, - .load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => { - const src_limb_mcv: MCValue = if (src_info) |info| .{ - .indirect = .{ .reg = info.addr_reg, .off = off }, - } else switch (resolved_src_mcv) { - .register, .register_pair, .register_triple, .register_quadruple => .{ - .register = resolved_src_mcv.getRegs()[limb_i], - }, - .eflags, - .register_offset, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => switch (limb_i) { - 0 => resolved_src_mcv, - else => .{ .immediate = 0 }, - }, - .memory => |addr| .{ .memory = @bitCast(@as(i64, @bitCast(addr)) + off) }, - .indirect => |reg_off| .{ .indirect = .{ - .reg = reg_off.reg, - .off = reg_off.off + off, - } }, - .load_frame => |frame_addr| .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + off, - } }, - else => unreachable, - }; - const src_limb_reg = if (src_limb_mcv.isRegister()) - src_limb_mcv.getReg().? - else - try self.copyToTmpRegister(limb_ty, src_limb_mcv); - try self.asmMemoryRegister( - mir_limb_tag, - dst_limb_mem, - registerAlias(src_limb_reg, limb_abi_size), - ); - }, - } - } - }, - } -} - -/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv. -/// Does not support byte-size operands. -fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void { - const pt = self.pt; - const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu)); - try self.spillEflagsIfOccupied(); - switch (dst_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register_offset, - .register_overflow, - .register_mask, - .indirect_load_frame, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // unmodifiable destination - .register => |dst_reg| { - const alias_size = switch (abi_size) { - 1 => 4, - else => abi_size, - }; - const dst_alias = registerAlias(dst_reg, alias_size); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - switch (abi_size) { - 1 => try self.asmRegisterRegister(.{ ._, .movzx }, dst_reg.to32(), dst_reg.to8()), - else => {}, - } - - const resolved_src_mcv = switch (src_mcv) { - else => src_mcv, - .air_ref => |src_ref| try self.resolveInst(src_ref), - }; - switch (resolved_src_mcv) { - .none, - .unreach, - .dead, - .undef, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .register => |src_reg| { - switch (abi_size) { - 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()), - else => {}, - } - try self.asmRegisterRegister( - .{ .i_, .mul }, - dst_alias, - registerAlias(src_reg, alias_size), - ); - }, - .immediate => |imm| { - if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| { - try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst_alias, dst_alias, .s(small)); - } else { - const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); - return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg }); - } - }, - .register_offset, - .eflags, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => { - const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); - switch (abi_size) { - 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()), - else => {}, - } - try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size)); - }, - .memory, .indirect, .load_frame => switch (abi_size) { - 1 => { - const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); - try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()); - try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size)); - }, - else => try self.asmRegisterMemory( - .{ .i_, .mul }, - dst_alias, - switch (resolved_src_mcv) { - .memory => |addr| .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse - return self.asmRegisterRegister( - .{ .i_, .mul }, - dst_alias, - registerAlias( - try self.copyToTmpRegister(dst_ty, resolved_src_mcv), - abi_size, - ), - ), - } }, - }, - .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = reg_off.off, - } }, - }, - .load_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = frame_addr.off, - } }, - }, - else => unreachable, - }, - ), - }, - } - }, - .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented - .memory, - .indirect, - .load_frame, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => { - const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); - const tmp_mcv = MCValue{ .register = tmp_reg }; - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv); - try self.genCopy(dst_ty, dst_mcv, tmp_mcv, .{}); - }, - } -} - -fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const arg_index = for (self.args, 0..) |arg, arg_index| { - if (arg != .none) break arg_index; - } else unreachable; - const src_mcv = self.args[arg_index]; - self.args = self.args[arg_index + 1 ..]; - const result: MCValue = if (self.mod.strip and self.liveness.isUnused(inst)) .unreach else result: { - const arg_ty = self.typeOfIndex(inst); - switch (src_mcv) { - .register, .register_pair, .load_frame => { - for (src_mcv.getRegs()) |reg| self.register_manager.getRegAssumeFree(reg, inst); - break :result src_mcv; - }, - .indirect => |reg_off| { - self.register_manager.getRegAssumeFree(reg_off.reg, null); - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genCopy(arg_ty, dst_mcv, src_mcv, .{}); - break :result dst_mcv; - }, - .indirect_load_frame => |frame_addr| { - const dst_mcv = try self.allocRegOrMem(inst, false); - const ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); - defer self.register_manager.unlockReg(ptr_lock); - try self.genSetReg(ptr_reg, .usize, .{ .load_frame = frame_addr }, .{}); - try self.genCopy(arg_ty, dst_mcv, .{ .indirect = .{ .reg = ptr_reg } }, .{}); - break :result dst_mcv; - }, - .elementwise_args => |regs_frame_addr| { - try self.spillEflagsIfOccupied(); - - const fn_info = zcu.typeToFunc(self.fn_type).?; - const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc); - var prev_reg: Register = undefined; - for ( - param_int_regs[param_int_regs.len - regs_frame_addr.regs ..], - 0.., - ) |dst_reg, elem_index| { - assert(self.register_manager.isRegFree(dst_reg)); - if (elem_index > 0) { - try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index)); - try self.asmRegisterRegister( - .{ ._, .@"or" }, - dst_reg.to8(), - prev_reg.to8(), - ); - } - prev_reg = dst_reg; - } - - const prev_lock = if (regs_frame_addr.regs > 0) - self.register_manager.lockRegAssumeUnused(prev_reg) - else - null; - defer if (prev_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = try self.allocRegOrMem(inst, false); - if (regs_frame_addr.regs > 0) try self.asmMemoryRegister( - .{ ._, .mov }, - try dst_mcv.mem(self, .{ .size = .byte }), - prev_reg.to8(), - ); - try self.genInlineMemset( - dst_mcv.address().offset(@intFromBool(regs_frame_addr.regs > 0)), - .{ .immediate = 0 }, - .{ .immediate = arg_ty.abiSize(zcu) - @intFromBool(regs_frame_addr.regs > 0) }, - .{}, - ); - - const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); - defer self.register_manager.unlockReg(index_lock); - - try self.asmRegisterImmediate( - .{ ._, .mov }, - index_reg.to32(), - .u(regs_frame_addr.regs), - ); - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmMemoryImmediate(.{ ._, .cmp }, .{ - .base = .{ .frame = regs_frame_addr.frame_index }, - .mod = .{ .rm = .{ - .size = .byte, - .index = index_reg.to64(), - .scale = .@"8", - .disp = regs_frame_addr.frame_off - @as(u6, regs_frame_addr.regs) * 8, - } }, - }, Immediate.u(0)); - const unset = try self.asmJccReloc(.e, undefined); - try self.asmMemoryRegister( - .{ ._s, .bt }, - try dst_mcv.mem(self, .{ .size = .dword }), - index_reg.to32(), - ); - self.performReloc(unset); - if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, index_reg.to32()); - } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - index_reg.to32(), - .u(arg_ty.vectorLen(zcu)), - ); - _ = try self.asmJccReloc(.b, loop); - - break :result dst_mcv; - }, - else => return self.fail("TODO implement arg for {f}", .{src_mcv}), - } - }; - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn genLocalDebugInfo(cg: *CodeGen, air_tag: Air.Inst.Tag, ty: Type, mcv: MCValue) !void { - assert(!cg.mod.strip); - _ = switch (air_tag) { - else => unreachable, - .arg, .dbg_var_val, .dbg_arg_inline => switch (mcv) { - .none, .unreach, .dead, .elementwise_args, .reserved_frame, .air_ref => unreachable, - .immediate => |imm| if (std.math.cast(u32, imm)) |small| try cg.addInst(.{ - .tag = .pseudo, - .ops = switch (air_tag) { - else => unreachable, - .arg, .dbg_arg_inline => .pseudo_dbg_arg_i_u, - .dbg_var_val => .pseudo_dbg_var_i_u, - }, - .data = .{ .i = .{ .i = small } }, - }) else try cg.addInst(.{ - .tag = .pseudo, - .ops = switch (air_tag) { - else => unreachable, - .arg, .dbg_arg_inline => .pseudo_dbg_arg_i_64, - .dbg_var_val => .pseudo_dbg_var_i_64, - }, - .data = .{ .i64 = imm }, - }), - .lea_frame => |frame_addr| try cg.addInst(.{ - .tag = .pseudo, - .ops = switch (air_tag) { - else => unreachable, - .arg, .dbg_arg_inline => .pseudo_dbg_arg_fa, - .dbg_var_val => .pseudo_dbg_var_fa, - }, - .data = .{ .fa = frame_addr }, - }), - else => { - const frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu)); - try cg.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{}); - _ = try cg.addInst(.{ - .tag = .pseudo, - .ops = switch (air_tag) { - else => unreachable, - .arg, .dbg_arg_inline => .pseudo_dbg_arg_m, - .dbg_var_val => .pseudo_dbg_var_m, - }, - .data = .{ .x = .{ - .payload = try cg.addExtra(Mir.Memory.encode(.{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .qword } }, - })), - } }, - }); - }, - }, - .dbg_var_ptr => switch (mcv) { - else => unreachable, - .none, .unreach, .dead, .elementwise_args, .reserved_frame, .air_ref => unreachable, - .lea_frame => |frame_addr| try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_var_m, - .data = .{ .x = .{ - .payload = try cg.addExtra(Mir.Memory.encode(.{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = frame_addr.off, - } }, - })), - } }, - }), - .lea_nav => |nav| try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_var_m, - .data = .{ .x = .{ - .payload = try cg.addExtra(Mir.Memory.encode(.{ - .base = .{ .nav = nav }, - .mod = .{ .rm = .{ .size = .qword } }, - })), - } }, - }), - .lea_uav => |uav| try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_var_m, - .data = .{ .x = .{ - .payload = try cg.addExtra(Mir.Memory.encode(.{ - .base = .{ .uav = uav }, - .mod = .{ .rm = .{ .size = .qword } }, - })), - } }, - }), - .lea_lazy_sym => |lazy_sym| try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_var_m, - .data = .{ .x = .{ - .payload = try cg.addExtra(Mir.Memory.encode(.{ - .base = .{ .lazy_sym = lazy_sym }, - .mod = .{ .rm = .{ .size = .qword } }, - })), - } }, - }), - .lea_extern_func => |extern_func| try cg.addInst(.{ - .tag = .pseudo, - .ops = .pseudo_dbg_var_m, - .data = .{ .x = .{ - .payload = try cg.addExtra(Mir.Memory.encode(.{ - .base = .{ .extern_func = extern_func }, - .mod = .{ .rm = .{ .size = .qword } }, - })), - } }, - }), - }, - }; -} - -fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier, opts: CopyOptions) !void { - if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{}); - - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Call, pl_op.payload); - const arg_refs: []const Air.Inst.Ref = - @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); - - const ExpectedContents = extern struct { - tys: [32][@sizeOf(Type)]u8 align(@alignOf(Type)), - vals: [32][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)), - }; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - const allocator = stack.get(); - - const arg_tys = try allocator.alloc(Type, arg_refs.len); - defer allocator.free(arg_tys); - for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = self.typeOf(arg_ref); - - const arg_vals = try allocator.alloc(MCValue, arg_refs.len); - defer allocator.free(arg_vals); - for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref }; - - const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals, opts); - - var bt = self.liveness.iterateBigTomb(inst); - try self.feed(&bt, pl_op.operand); - for (arg_refs) |arg_ref| try self.feed(&bt, arg_ref); - - const result = if (self.liveness.isUnused(inst)) .unreach else ret; - return self.finishAirResult(inst, result); -} - -fn genCall(self: *CodeGen, info: union(enum) { - air: Air.Inst.Ref, - extern_func: struct { - return_type: InternPool.Index, - param_types: []const InternPool.Index, - sym: []const u8, - }, -}, arg_types: []const Type, args: []const MCValue, opts: CopyOptions) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const fn_ty = switch (info) { - .air => |callee| fn_info: { - const callee_ty = self.typeOf(callee); - break :fn_info switch (callee_ty.zigTypeTag(zcu)) { - .@"fn" => callee_ty, - .pointer => callee_ty.childType(zcu), - else => unreachable, - }; - }, - .extern_func => |extern_func| try pt.funcType(.{ - .param_types = extern_func.param_types, - .return_type = extern_func.return_type, - .cc = self.target.cCallingConvention().?, - }), - }; - const fn_info = zcu.typeToFunc(fn_ty).?; - - const ExpectedContents = extern struct { - var_args: [32][@sizeOf(Type)]u8 align(@alignOf(Type)), - frame_indices: [32]FrameIndex, - reg_locks: [32][@sizeOf(?RegisterLock)]u8 align(@alignOf(?RegisterLock)), - }; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - const allocator = stack.get(); - - const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len); - defer allocator.free(var_args); - for (var_args, arg_types[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; - - const frame_indices = try allocator.alloc(FrameIndex, args.len); - defer allocator.free(frame_indices); - - var reg_locks: std.array_list.Managed(?RegisterLock) = .init(allocator); - defer reg_locks.deinit(); - try reg_locks.ensureTotalCapacity(16); - defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); - - var call_info = try self.resolveCallingConventionValues(fn_info, var_args, .call_frame); - defer call_info.deinit(self); - - // We need a properly aligned and sized call frame to be able to call this function. - { - const needed_call_frame: FrameAlloc = .init(.{ - .size = call_info.stack_byte_count, - .alignment = call_info.stack_align, - }); - const frame_allocs_slice = self.frame_allocs.slice(); - const stack_frame_size = - &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; - stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size); - const stack_frame_align = - &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; - stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align); - } - - try self.spillEflagsIfOccupied(); - try self.spillCallerPreservedRegs(fn_info.cc, call_info.err_ret_trace_reg); - - // set stack arguments first because this can clobber registers - // also clobber spill arguments as we go - switch (call_info.return_value.long) { - .none, .unreach => {}, - .indirect => |reg_off| try self.register_manager.getReg(reg_off.reg, null), - else => unreachable, - } - for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index| - switch (dst_arg) { - .none => {}, - .register => |reg| { - try self.register_manager.getReg(reg, null); - try reg_locks.append(self.register_manager.lockReg(reg)); - }, - .register_pair => |regs| { - for (regs) |reg| try self.register_manager.getReg(reg, null); - try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs)); - }, - .indirect => |reg_off| { - frame_index.* = try self.allocFrameIndex(.initType(arg_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); - try self.register_manager.getReg(reg_off.reg, null); - try reg_locks.append(self.register_manager.lockReg(reg_off.reg)); - }, - .load_frame => { - try self.genCopy(arg_ty, dst_arg, src_arg, opts); - try self.freeValue(src_arg, .{}); - }, - .elementwise_args => |regs_frame_addr| { - const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); - defer self.register_manager.unlockReg(index_lock); - - const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, switch (src_arg) { - else => src_arg, - .air_ref => |src_ref| try self.resolveInst(src_ref), - }.address()) }, - .mod = .{ .rm = .{ .size = .dword } }, - }; - const src_lock = switch (src_mem.base) { - .reg => |src_reg| self.register_manager.lockReg(src_reg), - else => null, - }; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - try self.asmRegisterImmediate( - .{ ._, .mov }, - index_reg.to32(), - .u(regs_frame_addr.regs), - ); - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32()); - try self.asmSetccMemory(.c, .{ - .base = .{ .frame = regs_frame_addr.frame_index }, - .mod = .{ .rm = .{ - .size = .byte, - .index = index_reg.to64(), - .scale = .@"8", - .disp = regs_frame_addr.frame_off - @as(u6, regs_frame_addr.regs) * 8, - } }, - }); - if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); - } else { - try self.asmRegister(.{ ._c, .in }, index_reg.to32()); - } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - index_reg.to32(), - .u(arg_ty.vectorLen(zcu)), - ); - _ = try self.asmJccReloc(.b, loop); - - const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc); - for (param_int_regs[param_int_regs.len - regs_frame_addr.regs ..]) |dst_reg| { - try self.register_manager.getReg(dst_reg, null); - try reg_locks.append(self.register_manager.lockReg(dst_reg)); - } - }, - else => unreachable, - }; - - if (call_info.err_ret_trace_reg != .none) { - if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { - if (switch (err_ret_trace.short) { - .register => |reg| call_info.err_ret_trace_reg != reg, - else => true, - }) { - try self.register_manager.getReg(call_info.err_ret_trace_reg, err_ret_trace_index); - try reg_locks.append(self.register_manager.lockReg(call_info.err_ret_trace_reg)); - - try self.genSetReg(call_info.err_ret_trace_reg, .usize, err_ret_trace.short, .{}); - err_ret_trace.trackMaterialize(err_ret_trace_index, .{ - .long = err_ret_trace.long, - .short = .{ .register = call_info.err_ret_trace_reg }, - }); - } - } - } - - // now we are free to set register arguments - switch (call_info.return_value.long) { - .none, .unreach => {}, - .indirect => |reg_off| { - const ret_ty: Type = .fromInterned(fn_info.return_type); - const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu)); - try self.genSetReg(reg_off.reg, .usize, .{ - .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, - }, .{}); - call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } }; - try reg_locks.append(self.register_manager.lockReg(reg_off.reg)); - }, - else => unreachable, - } - - for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| - switch (dst_arg) { - .none, .load_frame => {}, - .register => |dst_reg| switch (fn_info.cc) { - else => try self.genSetReg(registerAlias( - dst_reg, - @intCast(arg_ty.abiSize(zcu)), - ), arg_ty, src_arg, opts), - .x86_64_sysv, .x86_64_win => { - const promoted_ty = self.promoteInt(arg_ty); - const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(zcu)); - const dst_alias = registerAlias(dst_reg, promoted_abi_size); - try self.genSetReg(dst_alias, promoted_ty, src_arg, opts); - if (promoted_ty.toIntern() != arg_ty.toIntern()) - try self.truncateRegister(arg_ty, dst_alias); - }, - }, - .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, opts), - .indirect => |reg_off| try self.genSetReg(reg_off.reg, .usize, .{ - .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, - }, .{}), - .elementwise_args => |regs_frame_addr| { - const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ - .base = .{ .reg = try self.copyToTmpRegister( - .usize, - switch (src_arg) { - else => src_arg, - .air_ref => |src_ref| try self.resolveInst(src_ref), - }.address(), - ) }, - .mod = .{ .rm = .{ .size = .dword } }, - }; - const src_lock = switch (src_mem.base) { - .reg => |src_reg| self.register_manager.lockReg(src_reg), - else => null, - }; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc); - for ( - param_int_regs[param_int_regs.len - regs_frame_addr.regs ..], - 0.., - ) |dst_reg, elem_index| { - try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); - try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index)); - try self.asmSetccRegister(.c, dst_reg.to8()); - } - }, - else => unreachable, - }; - - if (fn_info.is_var_args) try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count)); - - // Due to incremental compilation, how function calls are generated depends - // on linking. - switch (info) { - .air => |callee| if (try self.air.value(callee, pt)) |func_value| { - const func_key = ip.indexToKey(func_value.ip_index); - switch (switch (func_key) { - else => func_key, - .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { - .nav => |nav| ip.indexToKey(zcu.navValue(nav).toIntern()), - else => func_key, - } else func_key, - }) { - else => unreachable, - .func => |func| try self.asmImmediate(.{ ._, .call }, .{ .nav = .{ .index = func.owner_nav } }), - .@"extern" => |@"extern"| try self.asmImmediate(.{ ._, .call }, .{ .nav = .{ .index = @"extern".owner_nav } }), - } - } else { - assert(self.typeOf(callee).zigTypeTag(zcu) == .pointer); - const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc); - try self.genSetReg(scratch_reg, .usize, .{ .air_ref = callee }, .{}); - try self.asmRegister(.{ ._, .call }, scratch_reg); - }, - .extern_func => |extern_func| try self.asmImmediate(.{ ._, .call }, .{ .extern_func = try self.addString(extern_func.sym) }), - } - return call_info.return_value.short; -} - -fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { - const pt = self.pt; - const zcu = pt.zcu; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - - const ret_ty = self.fn_type.fnReturnType(zcu); - switch (self.ret_mcv.short) { - .none => {}, - .register => |reg| { - const reg_lock = self.register_manager.lockRegAssumeUnused(reg); - defer self.register_manager.unlockReg(reg_lock); - try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); - }, - inline .register_pair, .register_triple, .register_quadruple => |regs| { - const reg_locks = self.register_manager.lockRegsAssumeUnused(regs.len, regs); - defer for (reg_locks) |reg_lock| self.register_manager.unlockReg(reg_lock); - try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); - }, - .indirect => |reg_off| { - try self.register_manager.getReg(reg_off.reg, null); - const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg); - defer self.register_manager.unlockReg(lock); - - try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{}); - try self.genSetMem( - .{ .reg = reg_off.reg }, - reg_off.off, - ret_ty, - .{ .air_ref = un_op }, - .{ .safety = safety }, - ); - }, - else => unreachable, - } - self.ret_mcv.liveOut(self, inst); - - if (self.err_ret_trace_reg != .none) { - if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { - if (switch (err_ret_trace.short) { - .register => |reg| self.err_ret_trace_reg != reg, - else => true, - }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{}); - err_ret_trace.liveOut(self, err_ret_trace_index); - } - } - - try self.finishAir(inst, .unreach, .{ un_op, .none, .none }); - - // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction - // which is available if the jump is 127 bytes or less forward. - const jmp_reloc = try self.asmJmpReloc(undefined); - try self.epilogue_relocs.append(self.gpa, jmp_reloc); -} - -fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ptr = try self.resolveInst(un_op); - - const ptr_ty = self.typeOf(un_op); - switch (self.ret_mcv.short) { - .none => {}, - .register, .register_pair => try self.load(self.ret_mcv.short, ptr_ty, ptr), - .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr, .{}), - else => unreachable, - } - self.ret_mcv.liveOut(self, inst); - - if (self.err_ret_trace_reg != .none) { - if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { - if (switch (err_ret_trace.short) { - .register => |reg| self.err_ret_trace_reg != reg, - else => true, - }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{}); - err_ret_trace.liveOut(self, err_ret_trace_index); - } - } - - try self.finishAir(inst, .unreach, .{ un_op, .none, .none }); - - // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction - // which is available if the jump is 127 bytes or less forward. - const jmp_reloc = try self.asmJmpReloc(undefined); - try self.epilogue_relocs.append(self.gpa, jmp_reloc); -} - -fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - var ty = self.typeOf(bin_op.lhs); - var null_compare: ?Mir.Inst.Index = null; - - const result: Condition = result: { - try self.spillEflagsIfOccupied(); - - const lhs_mcv = try self.resolveInst(bin_op.lhs); - const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) { - .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null }, - .register_pair => |lhs_regs| locks: { - const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs); - break :locks .{ locks[0], locks[1] }; - }, - .register_offset => |lhs_ro| .{ - self.register_manager.lockRegAssumeUnused(lhs_ro.reg), - null, - }, - else => @splat(null), - }; - defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(bin_op.rhs); - const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) { - .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null }, - .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs), - .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null }, - else => @splat(null), - }; - defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - switch (ty.zigTypeTag(zcu)) { - .float => { - const float_bits = ty.floatBits(self.target); - if (!switch (float_bits) { - 16 => self.hasFeature(.f16c), - 32 => self.hasFeature(.sse), - 64 => self.hasFeature(.sse2), - 80, 128 => false, - else => unreachable, - }) { - var sym_buf: ["__???f2".len]u8 = undefined; - const ret = try self.genCall(.{ .extern_func = .{ - .return_type = .i32_type, - .param_types = &.{ ty.toIntern(), ty.toIntern() }, - .sym = std.fmt.bufPrint(&sym_buf, "__{s}{c}f2", .{ - switch (op) { - .eq => "eq", - .neq => "ne", - .lt => "lt", - .lte => "le", - .gt => "gt", - .gte => "ge", - }, - floatCompilerRtAbiName(float_bits), - }) catch unreachable, - } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, .{}); - try self.genBinOpMir(.{ ._, .@"test" }, .i32, ret, ret); - break :result switch (op) { - .eq => .e, - .neq => .ne, - .lt => .l, - .lte => .le, - .gt => .g, - .gte => .ge, - }; - } - }, - .optional => if (!ty.optionalReprIsPayload(zcu)) { - const opt_ty = ty; - const opt_abi_size: u31 = @intCast(opt_ty.abiSize(zcu)); - ty = opt_ty.optionalChild(zcu); - const payload_abi_size: u31 = @intCast(ty.abiSize(zcu)); - - const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg); - defer self.register_manager.unlockReg(temp_lhs_lock); - - if (lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ ._, .mov }, - temp_lhs_reg.to8(), - try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }), - ) else { - try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{}); - try self.asmRegisterImmediate( - .{ ._r, .sh }, - registerAlias(temp_lhs_reg, opt_abi_size), - .u(payload_abi_size * 8), - ); - } - - const payload_compare = payload_compare: { - if (rhs_mcv.isBase()) { - const rhs_mem = - try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }); - try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8()); - const payload_compare = try self.asmJccReloc(.nz, undefined); - try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem); - break :payload_compare payload_compare; - } - - const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv); - const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg); - defer self.register_manager.unlockReg(temp_rhs_lock); - - try self.asmRegisterImmediate( - .{ ._r, .sh }, - registerAlias(temp_rhs_reg, opt_abi_size), - .u(payload_abi_size * 8), - ); - try self.asmRegisterRegister( - .{ ._, .@"test" }, - temp_lhs_reg.to8(), - temp_rhs_reg.to8(), - ); - const payload_compare = try self.asmJccReloc(.nz, undefined); - try self.asmRegisterRegister( - .{ ._, .cmp }, - temp_lhs_reg.to8(), - temp_rhs_reg.to8(), - ); - break :payload_compare payload_compare; - }; - null_compare = try self.asmJmpReloc(undefined); - self.performReloc(payload_compare); - }, - else => {}, - } - - switch (ty.zigTypeTag(zcu)) { - else => { - const abi_size: u16 = @intCast(ty.abiSize(zcu)); - const may_flip: enum { - may_flip, - must_flip, - must_not_flip, - } = if (abi_size > 8) switch (op) { - .lt, .gte => .must_not_flip, - .lte, .gt => .must_flip, - .eq, .neq => .may_flip, - } else .may_flip; - - const flipped = switch (may_flip) { - .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(), - .must_flip => true, - .must_not_flip => false, - }; - const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv; - const dst_mcv = if (unmat_dst_mcv.isRegister() or - (abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: { - const dst_mcv = try self.allocTempRegOrMem(ty, true); - try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{}); - break :dst dst_mcv; - }; - const dst_lock = - if (dst_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const src_mcv = try self.resolveInst(if (flipped) bin_op.lhs else bin_op.rhs); - const src_lock = - if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - break :result .fromCompareOperator( - if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned, - result_op: { - const flipped_op = if (flipped) op.reverse() else op; - if (abi_size > 8) switch (flipped_op) { - .lt, .gte => {}, - .lte, .gt => unreachable, - .eq, .neq => { - const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock }; - - const resolved_dst_mcv = switch (dst_mcv) { - else => dst_mcv, - .air_ref => |dst_ref| try self.resolveInst(dst_ref), - }; - const dst_info: OpInfo = switch (resolved_dst_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register_offset, - .register_overflow, - .register_mask, - .indirect, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .load_frame, - => null, - .memory, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => dst: { - switch (resolved_dst_mcv) { - .memory => |addr| if (std.math.cast( - i32, - @as(i64, @bitCast(addr)), - ) != null and std.math.cast( - i32, - @as(i64, @bitCast(addr)) + abi_size - 8, - ) != null) break :dst null, - .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, - else => unreachable, - } - - const dst_addr_reg = (try self.register_manager.allocReg( - null, - abi.RegisterClass.gp, - )).to64(); - const dst_addr_lock = - self.register_manager.lockRegAssumeUnused(dst_addr_reg); - errdefer self.register_manager.unlockReg(dst_addr_lock); - - try self.genSetReg(dst_addr_reg, .usize, resolved_dst_mcv.address(), .{}); - break :dst .{ - .addr_reg = dst_addr_reg, - .addr_lock = dst_addr_lock, - }; - }, - }; - defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock); - - const resolved_src_mcv = switch (src_mcv) { - else => src_mcv, - .air_ref => |src_ref| try self.resolveInst(src_ref), - }; - const src_info: OpInfo = switch (resolved_src_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register, - .register_offset, - .register_overflow, - .register_mask, - .indirect, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .register_pair, - .register_triple, - .register_quadruple, - .load_frame, - => null, - .memory, - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => src: { - switch (resolved_src_mcv) { - .memory => |addr| if (std.math.cast( - i32, - @as(i64, @bitCast(addr)), - ) != null and std.math.cast( - i32, - @as(i64, @bitCast(addr)) + abi_size - 8, - ) != null) break :src null, - .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, - else => unreachable, - } - - const src_addr_reg = (try self.register_manager.allocReg( - null, - abi.RegisterClass.gp, - )).to64(); - const src_addr_lock = - self.register_manager.lockRegAssumeUnused(src_addr_reg); - errdefer self.register_manager.unlockReg(src_addr_lock); - - try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{}); - break :src .{ - .addr_reg = src_addr_reg, - .addr_lock = src_addr_lock, - }; - }, - }; - defer if (src_info) |info| - self.register_manager.unlockReg(info.addr_lock); - - const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); - const acc_reg = regs[0].to64(); - const locks = self.register_manager.lockRegsAssumeUnused(2, regs); - defer for (locks) |lock| self.register_manager.unlockReg(lock); - - const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable; - var limb_i: u16 = 0; - while (limb_i < limbs_len) : (limb_i += 1) { - const off = limb_i * 8; - const tmp_reg = regs[@min(limb_i, 1)].to64(); - - try self.genSetReg(tmp_reg, .usize, if (dst_info) |info| .{ - .indirect = .{ .reg = info.addr_reg, .off = off }, - } else switch (resolved_dst_mcv) { - inline .register_pair, - .register_triple, - .register_quadruple, - => |dst_regs| .{ .register = dst_regs[limb_i] }, - .memory => |dst_addr| .{ - .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off), - }, - .indirect => |reg_off| .{ .indirect = .{ - .reg = reg_off.reg, - .off = reg_off.off + off, - } }, - .load_frame => |frame_addr| .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + off, - } }, - else => unreachable, - }, .{}); - - try self.genBinOpMir( - .{ ._, .xor }, - .usize, - .{ .register = tmp_reg }, - if (src_info) |info| .{ - .indirect = .{ .reg = info.addr_reg, .off = off }, - } else switch (resolved_src_mcv) { - inline .register_pair, - .register_triple, - .register_quadruple, - => |src_regs| .{ .register = src_regs[limb_i] }, - .memory => |src_addr| .{ - .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off), - }, - .indirect => |reg_off| .{ .indirect = .{ - .reg = reg_off.reg, - .off = reg_off.off + off, - } }, - .load_frame => |frame_addr| .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + off, - } }, - else => unreachable, - }, - ); - - if (limb_i > 0) - try self.asmRegisterRegister(.{ ._, .@"or" }, acc_reg, tmp_reg); - } - assert(limbs_len >= 2); // use flags from or - break :result_op flipped_op; - }, - }; - try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv); - break :result_op flipped_op; - }, - ); - }, - .float => { - const flipped = switch (op) { - .lt, .lte => true, - .eq, .gte, .gt, .neq => false, - }; - - const dst_mcv = if (flipped) rhs_mcv else lhs_mcv; - const dst_reg = if (dst_mcv.isRegister()) - dst_mcv.getReg().? - else - try self.copyToTmpRegister(ty, dst_mcv); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - const src_mcv = if (flipped) lhs_mcv else rhs_mcv; - - switch (ty.floatBits(self.target)) { - 16 => { - assert(self.hasFeature(.f16c)); - const tmp1_reg = - (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); - const tmp1_mcv = MCValue{ .register = tmp1_reg }; - const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg); - defer self.register_manager.unlockReg(tmp1_lock); - - const tmp2_reg = - (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); - const tmp2_mcv = MCValue{ .register = tmp2_reg }; - const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg); - defer self.register_manager.unlockReg(tmp2_lock); - - if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_w, .insr }, - tmp1_reg, - dst_reg.to128(), - try src_mcv.mem(self, .{ .size = .word }), - .u(1), - ) else try self.asmRegisterRegisterRegister( - .{ .vp_, .unpcklwd }, - tmp1_reg, - dst_reg.to128(), - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(ty, src_mcv)).to128(), - ); - try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, tmp1_reg, tmp1_reg); - try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg); - try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv); - }, - 32 => try self.genBinOpMir( - .{ ._ss, .ucomi }, - ty, - .{ .register = dst_reg }, - src_mcv, - ), - 64 => try self.genBinOpMir( - .{ ._sd, .ucomi }, - ty, - .{ .register = dst_reg }, - src_mcv, - ), - else => unreachable, - } - - break :result switch (if (flipped) op.reverse() else op) { - .lt, .lte => unreachable, // required to have been canonicalized to gt(e) - .gt => .a, - .gte => .ae, - .eq => .z_and_np, - .neq => .nz_or_p, - }; - }, - } - }; - - if (null_compare) |reloc| self.performReloc(reloc); - self.eflags_inst = inst; - return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; - const dst_mcv = try self.genBinOp( - inst, - .fromCmpOp(extra.compareOperator(), false), - extra.lhs, - extra.rhs, - ); - return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); -} - -fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); - const operand_ty = self.typeOf(pl_op.operand); - const result = try self.genTry(inst, pl_op.operand, body, operand_ty, false); - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); - const operand_ty = self.typeOf(extra.data.ptr); - const result = try self.genTry(inst, extra.data.ptr, body, operand_ty, true); - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn genTry( - self: *CodeGen, - inst: Air.Inst.Index, - operand: Air.Inst.Ref, - body: []const Air.Inst.Index, - operand_ty: Type, - operand_is_ptr: bool, -) !MCValue { - const liveness_cond_br = self.liveness.getCondBr(inst); - - const operand_mcv = try self.resolveInst(operand); - const is_err_mcv = if (operand_is_ptr) - try self.isErrPtr(null, operand_ty, operand_mcv) - else - try self.isErr(null, operand_ty, operand_mcv); - - const reloc = try self.genCondBrMir(.anyerror, is_err_mcv); - - if (self.liveness.operandDies(inst, 0)) { - if (operand.toIndex()) |operand_inst| try self.processDeath(operand_inst, .{}); - } - - const state = try self.saveState(); - - for (liveness_cond_br.else_deaths) |death| try self.processDeath(death, .{}); - try self.genBodyBlock(body); - try self.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - self.performReloc(reloc); - - for (liveness_cond_br.then_deaths) |death| try self.processDeath(death, .{}); - - const result = if (self.liveness.isUnused(inst)) - .unreach - else if (operand_is_ptr) - try self.genUnwrapErrUnionPayloadPtrMir(inst, operand_ty, operand_mcv) - else - try self.genUnwrapErrUnionPayloadMir(inst, operand_ty, operand_mcv); - return result; -} - -fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index { - const pt = self.pt; - const abi_size = ty.abiSize(pt.zcu); - switch (mcv) { - .eflags => |cc| { - // Here we map the opposites since the jump is to the false branch. - return self.asmJccReloc(cc.negate(), undefined); - }, - .register => |reg| { - try self.spillEflagsIfOccupied(); - try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1)); - return self.asmJccReloc(.z, undefined); - }, - .immediate, - .load_frame, - => { - try self.spillEflagsIfOccupied(); - if (abi_size <= 8) { - const reg = try self.copyToTmpRegister(ty, mcv); - return self.genCondBrMir(ty, .{ .register = reg }); - } - return self.fail("TODO implement condbr when condition is {f} with abi larger than 8 bytes", .{mcv}); - }, - else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}), - } -} - -fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const cond = try self.resolveInst(pl_op.operand); - const cond_ty = self.typeOf(pl_op.operand); - const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = - @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = - @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); - const liveness_cond_br = self.liveness.getCondBr(inst); - - // If the condition dies here in this condbr instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - if (self.liveness.operandDies(inst, 0)) { - if (pl_op.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); - } - - const state = try self.saveState(); - const reloc = try self.genCondBrMir(cond_ty, cond); - - for (liveness_cond_br.then_deaths) |death| try self.processDeath(death, .{}); - try self.genBodyBlock(then_body); - try self.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - self.performReloc(reloc); - - for (liveness_cond_br.else_deaths) |death| try self.processDeath(death, .{}); - try self.genBodyBlock(else_body); - try self.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - // We already took care of pl_op.operand earlier, so there's nothing left to do. -} - -fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const err_ty = eu_ty.errorUnionSet(zcu); - if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false - - try self.spillEflagsIfOccupied(); - - const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); - switch (eu_mcv) { - .register => |reg| { - const eu_lock = self.register_manager.lockReg(reg); - defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); - - const tmp_reg = try self.copyToTmpRegister(eu_ty, eu_mcv); - if (err_off > 0) { - try self.genShiftBinOpMir( - .{ ._r, .sh }, - eu_ty, - .{ .register = tmp_reg }, - .u8, - .{ .immediate = @as(u6, @intCast(err_off * 8)) }, - ); - } else { - try self.truncateRegister(.anyerror, tmp_reg); - } - try self.genBinOpMir(.{ ._, .cmp }, .anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 }); - }, - .load_frame => |frame_addr| try self.genBinOpMir( - .{ ._, .cmp }, - .anyerror, - .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + err_off, - } }, - .{ .immediate = 0 }, - ), - else => return self.fail("TODO implement isErr for {f}", .{eu_mcv}), - } - - if (maybe_inst) |inst| self.eflags_inst = inst; - return MCValue{ .eflags = .a }; -} - -fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const eu_ty = ptr_ty.childType(zcu); - const err_ty = eu_ty.errorUnionSet(zcu); - if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false - - try self.spillEflagsIfOccupied(); - - const ptr_reg = switch (ptr_mcv) { - .register => |reg| reg, - else => try self.copyToTmpRegister(ptr_ty, ptr_mcv), - }; - const ptr_lock = self.register_manager.lockReg(ptr_reg); - defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); - - const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); - try self.asmMemoryImmediate( - .{ ._, .cmp }, - .{ - .base = .{ .reg = ptr_reg }, - .mod = .{ .rm = .{ - .size = self.memSize(.anyerror), - .disp = err_off, - } }, - }, - .u(0), - ); - - if (maybe_inst) |inst| self.eflags_inst = inst; - return MCValue{ .eflags = .a }; -} - -fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { - // A block is a setup to be able to jump to the end. - const inst_tracking_i = self.inst_tracking.count(); - self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach)); - - try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() }); - const liveness = self.liveness.getBlock(inst); - - try self.genBody(body); - - var block_data = self.blocks.fetchRemove(inst).?; - defer block_data.value.deinit(self.gpa); - if (block_data.value.relocs.items.len > 0) { - var last_inst: Mir.Inst.Index = @intCast(self.mir_instructions.len - 1); - while (block_data.value.relocs.getLastOrNull() == last_inst) { - block_data.value.relocs.items.len -= 1; - self.mir_instructions.set(last_inst, .{ - .tag = .pseudo, - .ops = .pseudo_dead_none, - .data = undefined, - }); - last_inst -= 1; - } - for (block_data.value.relocs.items) |block_reloc| self.performReloc(block_reloc); - try self.restoreState(block_data.value.state, liveness.deaths, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - } - - if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i); - const tracking = &self.inst_tracking.values()[inst_tracking_i]; - if (self.liveness.isUnused(inst)) try tracking.die(self, inst, .{}); - self.getValueIfFree(tracking.short, inst); -} - -fn lowerSwitchBr( - cg: *CodeGen, - inst: Air.Inst.Index, - switch_br: Air.UnwrappedSwitch, - condition: MCValue, - condition_dies: bool, - is_loop: bool, -) !void { - const zcu = cg.pt.zcu; - const condition_ty = cg.typeOf(switch_br.operand); - const unsigned_condition_ty = try cg.pt.intType(.unsigned, cg.intInfo(condition_ty).?.bits); - - const ExpectedContents = extern struct { - liveness_deaths: [1 << 8 | 1]Air.Inst.Index, - bigint_limbs: [std.math.big.int.calcTwosCompLimbCount(1 << 10)]std.math.big.Limb, - relocs: [1 << 6]Mir.Inst.Index, - }; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); - const allocator = stack.get(); - - const state = try cg.saveState(); - - const liveness = try cg.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1); - defer allocator.free(liveness.deaths); - - if (!cg.mod.pic and cg.target.ofmt == .elf) table: { - var prong_items: u32 = 0; - var min: ?Value = null; - var max: ?Value = null; - { - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - prong_items += @intCast(case.items.len + case.ranges.len); - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val; - if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val; - } - for (case.ranges) |range| { - const low = Value.fromInterned(range[0].toInterned().?); - if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low; - const high = Value.fromInterned(range[1].toInterned().?); - if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high; - } - } - } - // This condition also triggers for switches with no non-else prongs and switches on bool. - if (prong_items < 1 << 2 or prong_items > 1 << 8) break :table; - - var min_space: Value.BigIntSpace = undefined; - const min_bigint = min.?.toBigInt(&min_space, zcu); - var max_space: Value.BigIntSpace = undefined; - const max_bigint = max.?.toBigInt(&max_space, zcu); - const limbs = try allocator.alloc( - std.math.big.Limb, - @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1, - ); - defer allocator.free(limbs); - const table_len = table_len: { - var table_len_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - table_len_bigint.sub(max_bigint, min_bigint); - assert(table_len_bigint.positive); // min <= max - break :table_len @as(u11, table_len_bigint.toConst().toInt(u10) catch break :table) + 1; // no more than a 1024 entry table - }; - assert(prong_items <= table_len); // each prong item introduces at least one unique integer to the range - if (prong_items < table_len >> 2) break :table; // no more than 75% waste - - const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: { - const condition_index = try cg.allocTempRegOrMem(condition_ty, true); - try cg.genCopy(condition_ty, condition_index, condition, .{}); - break :condition_index condition_index; - }; - try cg.spillEflagsIfOccupied(); - if (min.?.orderAgainstZero(zcu).compare(.neq)) try cg.genBinOpMir( - .{ ._, .sub }, - condition_ty, - condition_index, - .{ .air_ref = Air.internedToRef(min.?.toIntern()) }, - ); - const else_reloc = if (switch_br.else_body_len > 0) else_reloc: { - var cond_temp = try cg.tempInit(unsigned_condition_ty, condition_index); - var table_max_temp = try cg.tempFromValue(try cg.pt.intValue(unsigned_condition_ty, table_len - 1)); - const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, cg) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try cond_temp.die(cg); - try table_max_temp.die(cg); - const else_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); - try cc_temp.die(cg); - break :else_reloc else_reloc; - } else undefined; - const table_start: u31 = @intCast(cg.mir_table.items.len); - { - const condition_index_reg = condition_index_reg: { - if (condition_index.isRegister()) { - const condition_index_reg = condition_index.getReg().?; - if (condition_index_reg.isClass(.general_purpose)) break :condition_index_reg condition_index_reg; - } - break :condition_index_reg try cg.copyToTmpRegister(.usize, condition_index); - }; - const condition_index_lock = cg.register_manager.lockReg(condition_index_reg); - defer if (condition_index_lock) |lock| cg.register_manager.unlockReg(lock); - try cg.truncateRegister(condition_ty, condition_index_reg); - const ptr_size = @divExact(cg.target.ptrBitWidth(), 8); - try cg.asmMemory(.{ ._mp, .j }, .{ - .base = .table, - .mod = .{ .rm = .{ - .size = .ptr, - .index = condition_index_reg.toSize(.ptr, cg.target), - .scale = .fromFactor(@intCast(ptr_size)), - .disp = table_start * ptr_size, - } }, - }); - } - const else_reloc_marker: u32 = 0; - assert(cg.mir_instructions.len > else_reloc_marker); - try cg.mir_table.appendNTimes(cg.gpa, else_reloc_marker, table_len); - if (is_loop) try cg.loop_switches.putNoClobber(cg.gpa, inst, .{ - .start = table_start, - .len = table_len, - .min = min.?, - .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable", - }); - defer if (is_loop) { - var loop_switch_data = cg.loop_switches.fetchRemove(inst).?.value; - switch (loop_switch_data.else_relocs) { - .@"unreachable", .backward => {}, - .forward => |*else_relocs| else_relocs.deinit(cg.gpa), - } - }; - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - { - const table = cg.mir_table.items[table_start..][0..table_len]; - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - var val_space: Value.BigIntSpace = undefined; - const val_bigint = val.toBigInt(&val_space, zcu); - var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - index_bigint.sub(val_bigint, min_bigint); - table[index_bigint.toConst().toInt(u10) catch unreachable] = @intCast(cg.mir_instructions.len); - } - for (case.ranges) |range| { - var low_space: Value.BigIntSpace = undefined; - const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu); - var high_space: Value.BigIntSpace = undefined; - const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu); - var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; - index_bigint.sub(low_bigint, min_bigint); - const start = index_bigint.toConst().toInt(u10) catch unreachable; - index_bigint.sub(high_bigint, min_bigint); - const end = @as(u11, index_bigint.toConst().toInt(u10) catch unreachable) + 1; - @memset(table[start..end], @intCast(cg.mir_instructions.len)); - } - } - - for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand, .{}); - - try cg.genBodyBlock(case.body); - try cg.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - } - if (switch_br.else_body_len > 0) { - const else_body = cases_it.elseBody(); - - const else_deaths = liveness.deaths.len - 1; - for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand, .{}); - - cg.performReloc(else_reloc); - if (is_loop) { - const loop_switch_data = cg.loop_switches.getPtr(inst).?; - for (loop_switch_data.else_relocs.forward.items) |reloc| cg.performReloc(reloc); - loop_switch_data.else_relocs.forward.deinit(cg.gpa); - loop_switch_data.else_relocs = .{ .backward = @intCast(cg.mir_instructions.len) }; - } - for (cg.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) { - entry.* = @intCast(cg.mir_instructions.len); - }; - - try cg.genBodyBlock(else_body); - try cg.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - } - return; - } - - var cases_it = switch_br.iterateCases(); - while (cases_it.next()) |case| { - const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); - defer allocator.free(relocs); - - try cg.spillEflagsIfOccupied(); - for (case.items, relocs[0..case.items.len]) |item, *reloc| { - const item_mcv = try cg.resolveInst(item); - const cc: Condition = switch (condition) { - .eflags => |cc| switch (item_mcv.immediate) { - 0 => cc.negate(), - 1 => cc, - else => unreachable, - }, - else => cc: { - var cond_temp = try cg.tempInit(condition_ty, condition); - var item_temp = try cg.tempInit(condition_ty, item_mcv); - const cc_temp = cond_temp.cmpInts(.eq, &item_temp, cg) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try item_temp.die(cg); - const cc = cc_temp.tracking(cg).short.eflags; - try cc_temp.die(cg); - try cond_temp.die(cg); - try cg.resetTemps(@enumFromInt(0)); - break :cc cc; - }, - }; - reloc.* = try cg.asmJccReloc(cc, undefined); - } - - for (case.ranges, relocs[case.items.len..]) |range, *reloc| { - var cond_temp = try cg.tempInit(condition_ty, condition); - const min_mcv = try cg.resolveInst(range[0]); - const max_mcv = try cg.resolveInst(range[1]); - // `null` means always false. - const lt_min = cc: switch (condition) { - .eflags => |cc| switch (min_mcv.immediate) { - 0 => null, // condition never <0 - 1 => cc.negate(), - else => unreachable, - }, - else => { - var min_temp = try cg.tempInit(condition_ty, min_mcv); - const cc_temp = cond_temp.cmpInts(.lt, &min_temp, cg) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try min_temp.die(cg); - const cc = cc_temp.tracking(cg).short.eflags; - try cc_temp.die(cg); - break :cc cc; - }, - }; - const lt_min_reloc = if (lt_min) |cc| r: { - break :r try cg.asmJccReloc(cc, undefined); - } else null; - // `null` means always true. - const lte_max = switch (condition) { - .eflags => |cc| switch (max_mcv.immediate) { - 0 => cc.negate(), - 1 => null, // condition always >=1 - else => unreachable, - }, - else => cc: { - var max_temp = try cg.tempInit(condition_ty, max_mcv); - const cc_temp = cond_temp.cmpInts(.lte, &max_temp, cg) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try max_temp.die(cg); - const cc = cc_temp.tracking(cg).short.eflags; - try cc_temp.die(cg); - break :cc cc; - }, - }; - try cond_temp.die(cg); - try cg.resetTemps(@enumFromInt(0)); - // "Success" case is in `reloc`.... - if (lte_max) |cc| { - reloc.* = try cg.asmJccReloc(cc, undefined); - } else { - reloc.* = try cg.asmJmpReloc(undefined); - } - // ...and "fail" case falls through to next checks. - if (lt_min_reloc) |r| cg.performReloc(r); - } - - cg.checkInvariantsAfterAirInst(); - - // The jump to skip this case if the conditions all failed. - const skip_case_reloc = try cg.asmJmpReloc(undefined); - - for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand, .{}); - - // Relocate all success cases to the body we're about to generate. - for (relocs) |reloc| cg.performReloc(reloc); - try cg.genBodyBlock(case.body); - try cg.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - - // Relocate the "skip" branch to fall through to the next case. - cg.performReloc(skip_case_reloc); - } - if (switch_br.else_body_len > 0) { - const else_body = cases_it.elseBody(); - - const else_deaths = liveness.deaths.len - 1; - for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand, .{}); - - try cg.genBodyBlock(else_body); - try cg.restoreState(state, &.{}, .{ - .emit_instructions = false, - .update_tracking = true, - .resurrect = true, - .close_scope = true, - }); - } -} - -fn airSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void { - const switch_br = self.air.unwrapSwitch(inst); - const condition = try self.resolveInst(switch_br.operand); - - // If the condition dies here in this switch instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - const condition_dies = self.liveness.operandDies(inst, 0); - if (condition_dies) { - if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); - } - try self.lowerSwitchBr(inst, switch_br, condition, condition_dies, false); - - // We already took care of pl_op.operand earlier, so there's nothing left to do -} - -fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void { - const switch_br = self.air.unwrapSwitch(inst); - const condition = try self.resolveInst(switch_br.operand); - - const mat_cond = if (condition.isModifiable() and - self.reuseOperand(inst, switch_br.operand, 0, condition)) - condition - else mat_cond: { - const mat_cond = try self.allocRegOrMem(inst, true); - try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{}); - break :mat_cond mat_cond; - }; - self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(mat_cond)); - - // If the condition dies here in this switch instruction, process - // that death now instead of later as this has an effect on - // whether it needs to be spilled in the branches - if (self.liveness.operandDies(inst, 0)) { - if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); - } - - // Ensure a register is available for dispatch. - if (!mat_cond.isRegister()) _ = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - - const state = try self.saveState(); - - try self.loops.putNoClobber(self.gpa, inst, .{ - .state = state, - .target = @intCast(self.mir_instructions.len), - }); - defer assert(self.loops.remove(inst)); - - // Stop tracking block result without forgetting tracking info - try self.freeValue(mat_cond, .{}); - - try self.lowerSwitchBr(inst, switch_br, mat_cond, true, true); - - try self.processDeath(inst, .{}); -} - -fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void { - const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; - - const block_ty = self.typeOfIndex(br.block_inst); - const loop_data = self.loops.getPtr(br.block_inst).?; - const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; - { - try self.getValue(block_tracking.short, null); - const src_mcv = try self.resolveInst(br.operand); - - if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { - try self.getValue(block_tracking.short, br.block_inst); - // .long = .none to avoid merging operand and block result stack frames. - const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; - try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*); - for (current_tracking.getRegs()) |src_reg| self.register_manager.freeReg(src_reg); - } else { - try self.getValue(block_tracking.short, br.block_inst); - try self.genCopy(block_ty, block_tracking.short, try self.resolveInst(br.operand), .{}); - } - } - - // Process operand death so that it is properly accounted for in the State below. - if (self.liveness.operandDies(inst, 0)) { - if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); - } - - try self.restoreState(loop_data.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - - if (self.loop_switches.getPtr(br.block_inst)) |table| { - const condition_ty = self.typeOf(br.operand); - const unsigned_condition_ty = try self.pt.intType(.unsigned, self.intInfo(condition_ty).?.bits); - const condition_mcv = block_tracking.short; - try self.spillEflagsIfOccupied(); - if (table.min.orderAgainstZero(self.pt.zcu).compare(.neq)) try self.genBinOpMir( - .{ ._, .sub }, - condition_ty, - condition_mcv, - .{ .air_ref = Air.internedToRef(table.min.toIntern()) }, - ); - switch (table.else_relocs) { - .@"unreachable" => {}, - .forward => |*else_relocs| { - var cond_temp = try self.tempInit(unsigned_condition_ty, condition_mcv); - var table_max_temp = try self.tempFromValue(try self.pt.intValue(unsigned_condition_ty, table.len - 1)); - const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try cond_temp.die(self); - try table_max_temp.die(self); - try else_relocs.append(self.gpa, try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined)); - try cc_temp.die(self); - }, - .backward => |else_reloc| { - var cond_temp = try self.tempInit(unsigned_condition_ty, condition_mcv); - var table_max_temp = try self.tempFromValue(try self.pt.intValue(unsigned_condition_ty, table.len - 1)); - const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) { - error.SelectFailed => unreachable, - else => |e| return e, - }; - try cond_temp.die(self); - try table_max_temp.die(self); - _ = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, else_reloc); - try cc_temp.die(self); - }, - } - { - const condition_index_reg = if (condition_mcv.isRegister()) condition_mcv.getReg().? else cond: { - const condition_index_reg = - RegisterManager.regAtTrackedIndex(@intCast(loop_data.state.free_registers.findFirstSet().?)); - try self.genSetReg(condition_index_reg, condition_ty, condition_mcv, .{}); - break :cond condition_index_reg; - }; - const condition_index_lock = self.register_manager.lockReg(condition_index_reg); - defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock); - try self.truncateRegister(condition_ty, condition_index_reg); - const ptr_size = @divExact(self.target.ptrBitWidth(), 8); - try self.asmMemory(.{ ._mp, .j }, .{ - .base = .table, - .mod = .{ .rm = .{ - .size = .ptr, - .index = condition_index_reg.toSize(.ptr, self.target), - .scale = .fromFactor(@intCast(ptr_size)), - .disp = @intCast(table.start * ptr_size), - } }, - }); - } - - return self.finishAir(inst, .none, .{ br.operand, .none, .none }); - } - - // Emit a jump with a relocation. It will be patched up after the block ends. - // Leave the jump offset undefined - _ = try self.asmJmpReloc(loop_data.target); - - // Stop tracking block result without forgetting tracking info - try self.freeValue(block_tracking.short, .{ .emit_instructions = false }); -} - -fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void { - const next_inst: u32 = @intCast(self.mir_instructions.len); - switch (self.mir_instructions.items(.tag)[reloc]) { - .j => {}, - .pseudo => switch (self.mir_instructions.items(.ops)[reloc]) { - .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {}, - else => unreachable, - }, - .lea => switch (self.mir_instructions.items(.ops)[reloc]) { - .rm => { - const rx = self.mir_instructions.items(.data)[reloc].rx; - assert(rx.fixes == ._); - const mem_info: Mir.Memory.Info = @bitCast( - self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "info").?], - ); - assert(mem_info.base == .rip_inst); - self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "base").?] = next_inst; - return; - }, - else => unreachable, - }, - else => unreachable, - } - self.mir_instructions.items(.data)[reloc].inst.inst = next_inst; -} - -fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = self.pt.zcu; - const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; - - const block_ty = self.typeOfIndex(br.block_inst); - const block_unused = - !block_ty.hasRuntimeBitsIgnoreComptime(zcu) or self.liveness.isUnused(br.block_inst); - const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; - const block_data = self.blocks.getPtr(br.block_inst).?; - const first_br = block_data.relocs.items.len == 0; - const block_result = result: { - if (block_unused) break :result .none; - - if (!first_br) try self.getValue(block_tracking.short, null); - const src_mcv = try self.resolveInst(br.operand); - - if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { - if (first_br) break :result src_mcv; - - try self.getValue(block_tracking.short, br.block_inst); - try InstTracking.materializeUnsafe( - // .long = .none to avoid merging operand and block result stack frames. - .{ .long = .none, .short = src_mcv }, - self, - br.block_inst, - block_tracking.*, - ); - try self.freeValue(src_mcv, .{}); - break :result block_tracking.short; - } - - const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: { - try self.getValue(block_tracking.short, br.block_inst); - break :dst block_tracking.short; - }; - try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand), .{}); - break :result dst_mcv; - }; - - // Process operand death so that it is properly accounted for in the State below. - if (self.liveness.operandDies(inst, 0)) { - if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); - } - - if (first_br) { - block_tracking.* = .init(block_result); - try self.saveRetroactiveState(&block_data.state); - } else try self.restoreState(block_data.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - - // Emit a jump with a relocation. It will be patched up after the block ends. - // Leave the jump offset undefined - const jmp_reloc = try self.asmJmpReloc(undefined); - try block_data.relocs.append(self.gpa, jmp_reloc); - - // Stop tracking block result without forgetting tracking info - try self.freeValue(block_tracking.short, .{ .emit_instructions = false }); -} - -fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { - @setEvalBranchQuota(1_100); - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Asm, ty_pl.payload); - const outputs_len = extra.data.flags.outputs_len; - var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..outputs_len]); - extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); - extra_i += inputs.len; - - var result: MCValue = .none; - var args: std.array_list.Managed(MCValue) = .init(self.gpa); - try args.ensureTotalCapacity(outputs.len + inputs.len); - defer { - for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{ - .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue, - }); - args.deinit(); - } - var arg_map: std.StringHashMap(u8) = .init(self.gpa); - try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len)); - defer arg_map.deinit(); - - var outputs_extra_i = extra_i; - for (outputs) |output| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - const maybe_inst = switch (output) { - .none => inst, - else => null, - }; - const ty = switch (output) { - .none => self.typeOfIndex(inst), - else => self.typeOf(output).childType(zcu), - }; - const is_read = switch (constraint[0]) { - '=' => false, - '+' => read: { - if (output == .none) return self.fail( - "read-write constraint unsupported for asm result: '{s}'", - .{constraint}, - ); - break :read true; - }, - else => return self.fail("invalid constraint: '{s}'", .{constraint}), - }; - const is_early_clobber = constraint[1] == '&'; - const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..]; - const arg_mcv: MCValue = arg_mcv: { - const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or - std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x")) - registerAlias( - self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) { - 'r' => abi.RegisterClass.gp, - 'f' => abi.RegisterClass.x87, - 'x' => abi.RegisterClass.sse, - else => unreachable, - }) orelse return self.fail("ran out of registers lowering inline asm", .{}), - @intCast(ty.abiSize(zcu)), - ) - else if (std.mem.eql(u8, rest, "m")) - if (output != .none) null else return self.fail( - "memory constraint unsupported for asm result: '{s}'", - .{constraint}, - ) - else if (std.mem.eql(u8, rest, "g") or - std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or - std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r")) - self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse - if (output != .none) - null - else - return self.fail("ran out of registers lowering inline asm", .{}) - else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}")) - parseRegName(rest["{".len .. rest.len - "}".len]) orelse - return self.fail("invalid register constraint: '{s}'", .{constraint}) - else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { - const index = std.fmt.charToDigit(rest[0], 10) catch unreachable; - if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{ - constraint, - }); - break :arg_mcv args.items[index]; - } else return self.fail("invalid constraint: '{s}'", .{constraint}); - break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { - const ptr_mcv = try self.resolveInst(output); - switch (ptr_mcv) { - .immediate => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| - break :arg ptr_mcv.deref(), - .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(), - else => {}, - } - break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(.usize, ptr_mcv) } }; - }; - }; - if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |tracked_index| { - try self.register_manager.getRegIndex(tracked_index, if (output == .none) inst else null); - _ = self.register_manager.lockRegIndexAssumeUnused(tracked_index); - }; - if (!std.mem.eql(u8, name, "_")) - arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); - args.appendAssumeCapacity(arg_mcv); - if (output == .none) result = arg_mcv; - if (is_read) try self.load(arg_mcv, self.typeOf(output), .{ .air_ref = output }); - } - - for (inputs) |input| { - const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); - const constraint = std.mem.sliceTo(input_bytes, 0); - const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - const ty = self.typeOf(input); - const input_mcv = try self.resolveInst(input); - const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or - std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x")) - arg: { - const rc = switch (constraint[0]) { - 'r' => abi.RegisterClass.gp, - 'f' => abi.RegisterClass.x87, - 'x' => abi.RegisterClass.sse, - else => unreachable, - }; - if (input_mcv.isRegister() and - rc.isSet(RegisterManager.indexOfRegIntoTracked(input_mcv.getReg().?).?)) - break :arg input_mcv; - const reg = try self.register_manager.allocReg(null, rc); - try self.genSetReg(reg, ty, input_mcv, .{}); - break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) }; - } else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n")) - switch (input_mcv) { - .immediate => |imm| .{ .immediate = imm }, - else => return self.fail("immediate operand requires comptime value: '{s}'", .{ - constraint, - }), - } - else if (std.mem.eql(u8, constraint, "m")) arg: { - switch (input_mcv) { - .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| - break :arg input_mcv, - .indirect, .load_frame => break :arg input_mcv, - .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, - else => { - const temp_mcv = try self.allocTempRegOrMem(ty, false); - try self.genCopy(ty, temp_mcv, input_mcv, .{}); - break :arg temp_mcv; - }, - } - const addr_reg = self.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse { - const temp_mcv = try self.allocTempRegOrMem(ty, false); - try self.genCopy(ty, temp_mcv, input_mcv, .{}); - break :arg temp_mcv; - }; - try self.genSetReg(addr_reg, .usize, input_mcv.address(), .{}); - break :arg .{ .indirect = .{ .reg = addr_reg } }; - } else if (std.mem.eql(u8, constraint, "g") or - std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or - std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r")) - arg: { - switch (input_mcv) { - .register, .indirect, .load_frame => break :arg input_mcv, - .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| - break :arg input_mcv, - else => {}, - } - const temp_mcv = try self.allocTempRegOrMem(ty, true); - try self.genCopy(ty, temp_mcv, input_mcv, .{}); - break :arg temp_mcv; - } else if (std.mem.eql(u8, constraint, "X")) - input_mcv - else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: { - const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse - return self.fail("invalid register constraint: '{s}'", .{constraint}); - try self.register_manager.getReg(reg, null); - try self.genSetReg(reg, ty, input_mcv, .{}); - break :arg .{ .register = reg }; - } else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: { - const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable; - if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint}); - try self.genCopy(ty, args.items[index], input_mcv, .{}); - break :arg args.items[index]; - } else return self.fail("invalid constraint: '{s}'", .{constraint}); - if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { - _ = self.register_manager.lockReg(reg); - }; - if (!std.mem.eql(u8, name, "_")) - arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); - args.appendAssumeCapacity(arg_mcv); - } - - const ip = &zcu.intern_pool; - const aggregate = ip.indexToKey(extra.data.clobbers).aggregate; - const struct_type: Type = .fromInterned(aggregate.ty); - switch (aggregate.storage) { - .elems => |elems| for (elems, 0..) |elem, i| switch (elem) { - .bool_true => { - const clobber = struct_type.structFieldName(i, zcu).toSlice(ip).?; - assert(clobber.len != 0); - - if (std.mem.eql(u8, clobber, "memory") or - std.mem.eql(u8, clobber, "fpsr") or - std.mem.eql(u8, clobber, "fpcr") or - std.mem.eql(u8, clobber, "mxcsr") or - std.mem.eql(u8, clobber, "dirflag")) - { - // ok, sure - } else if (std.mem.eql(u8, clobber, "cc") or - std.mem.eql(u8, clobber, "flags") or - std.mem.eql(u8, clobber, "eflags") or - std.mem.eql(u8, clobber, "rflags")) - { - try self.spillEflagsIfOccupied(); - } else { - try self.register_manager.getReg(parseRegName(clobber) orelse - return self.fail("invalid clobber: '{s}'", .{clobber}), null); - } - }, - .bool_false => continue, - else => unreachable, - }, - .repeated_elem => |elem| switch (elem) { - .bool_true => @panic("TODO"), - .bool_false => {}, - else => unreachable, - }, - .bytes => @panic("TODO"), - } - - const Label = struct { - target: Mir.Inst.Index = undefined, - pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, - - const Kind = enum { definition, reference }; - - fn isValid(kind: Kind, name: []const u8) bool { - for (name, 0..) |c, i| switch (c) { - else => return false, - '$' => if (i == 0) return false, - '.' => {}, - '0'...'9' => if (i == 0) switch (kind) { - .definition => if (name.len != 1) return false, - .reference => { - if (name.len != 2) return false; - switch (name[1]) { - else => return false, - 'B', 'F', 'b', 'f' => {}, - } - }, - }, - '@', 'A'...'Z', '_', 'a'...'z' => {}, - }; - return name.len > 0; - } - }; - var labels: std.StringHashMapUnmanaged(Label) = .empty; - defer { - var label_it = labels.valueIterator(); - while (label_it.next()) |label| label.pending_relocs.deinit(self.gpa); - labels.deinit(self.gpa); - } - - const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; - var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;"); - next_line: while (line_it.next()) |line| { - var mnem_it = std.mem.tokenizeAny(u8, line, " \t"); - var prefix: encoder.Instruction.Prefix = .none; - const mnem_str = while (mnem_it.next()) |mnem_str| { - if (mnem_str[0] == '#') continue :next_line; - if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line; - if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) |pre| { - if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str}); - prefix = pre; - continue; - } - if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str; - const label_name = mnem_str[0 .. mnem_str.len - ":".len]; - if (!Label.isValid(.definition, label_name)) - return self.fail("invalid label: '{s}'", .{label_name}); - const label_gop = try labels.getOrPut(self.gpa, label_name); - if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else { - const anon = std.ascii.isDigit(label_name[0]); - if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0) - return self.fail("redefined label: '{s}'", .{label_name}); - for (label_gop.value_ptr.pending_relocs.items) |pending_reloc| - self.performReloc(pending_reloc); - if (anon) - label_gop.value_ptr.pending_relocs.clearRetainingCapacity() - else - label_gop.value_ptr.pending_relocs.clearAndFree(self.gpa); - } - label_gop.value_ptr.target = @intCast(self.mir_instructions.len); - } else continue; - if (mnem_str[0] == '.') { - if (prefix != .none) return self.fail("prefixed directive: '{s} {s}'", .{ @tagName(prefix), mnem_str }); - prefix = .directive; - } - - var mnem_size: struct { - op_has_size: std.StaticBitSet(4), - size: Memory.Size, - used: bool, - fn init(size: ?Memory.Size) @This() { - return .{ - .op_has_size = if (size) |_| .initFull() else .initEmpty(), - .size = size orelse .none, - .used = false, - }; - } - fn use(size: *@This(), op_index: usize) ?Memory.Size { - if (!size.op_has_size.isSet(op_index)) return null; - size.used = true; - return size.size; - } - } = .init(if (prefix == .directive) - null - else if (std.mem.endsWith(u8, mnem_str, "b")) - .byte - else if (std.mem.endsWith(u8, mnem_str, "w")) - .word - else if (std.mem.endsWith(u8, mnem_str, "l")) - .dword - else if (std.mem.endsWith(u8, mnem_str, "q") and - (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq"))) - .qword - else if (std.mem.endsWith(u8, mnem_str, "t")) - .tbyte - else - null); - var mnem_tag = while (true) break std.meta.stringToEnum( - encoder.Instruction.Mnemonic, - mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size.size != .none)], - ) orelse if (mnem_size.size != .none) { - mnem_size = .init(null); - continue; - } else return self.fail("invalid mnemonic: '{s}'", .{mnem_str}); - fixed_mnem_size: { - const fixed_mnem_size: Memory.Size = switch (mnem_tag) { - .clflush => .byte, - .fldcw, .fnstcw, .fstcw, .fnstsw, .fstsw => .word, - .fldenv, .fnstenv, .fstenv => .none, - .frstor, .fsave, .fnsave, .fxrstor, .fxrstor64, .fxsave, .fxsave64 => .none, - .in => { - mnem_size.op_has_size.unset(0); - break :fixed_mnem_size; - }, - .invlpg => .none, - .invpcid => .xword, - .ldmxcsr, .stmxcsr, .vldmxcsr, .vstmxcsr => .dword, - .out => { - mnem_size.op_has_size.unset(1); - break :fixed_mnem_size; - }, - else => break :fixed_mnem_size, - }; - if (mnem_size.size != .none and mnem_size.size != fixed_mnem_size) - return self.fail("invalid size: '{s}'", .{mnem_str}); - mnem_size = .init(fixed_mnem_size); - } - - var ops: [4]Operand = @splat(.none); - var ops_len: usize = 0; - - var last_op = false; - var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ','); - next_op: for (&ops, 0..) |*op, op_index| { - const op_str = while (!last_op) { - const full_str = op_it.next() orelse break :next_op; - const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse - std.mem.indexOf(u8, full_str, "//")) |comment| - code: { - last_op = true; - break :code full_str[0..comment]; - } else full_str; - const trim_str = std.mem.trim(u8, code_str, " \t*"); - if (trim_str.len > 0) break trim_str; - } else break; - if (std.mem.startsWith(u8, op_str, "%%")) { - const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':'); - const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse - return self.fail("invalid register: '{s}'", .{op_str}); - if (colon) |colon_pos| { - const disp = std.fmt.parseInt(i32, op_str[colon_pos + ":".len ..], 0) catch - return self.fail("invalid displacement: '{s}'", .{op_str}); - op.* = .{ .mem = .{ - .base = .{ .reg = reg }, - .mod = .{ .rm = .{ - .size = mnem_size.use(op_index) orelse - return self.fail("unknown size: '{s}'", .{op_str}), - .disp = disp, - } }, - } }; - } else { - if (mnem_size.use(op_index)) |size| if (reg.size().bitSize(self.target) != size.bitSize(self.target)) - return self.fail("invalid register size: '{s}'", .{op_str}); - op.* = .{ .reg = reg }; - } - } else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) { - const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':'); - const modifier = if (colon) |colon_pos| - op_str[colon_pos + ":".len .. op_str.len - "]".len] - else - ""; - op.* = switch (args.items[ - arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse - return self.fail("no matching constraint: '{s}'", .{op_str}) - ]) { - .immediate => |imm| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c")) - .{ .imm = .u(imm) } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .register => |reg| if (std.mem.eql(u8, modifier, "")) - .{ .reg = if (mnem_size.use(op_index)) |size| reg.toSize(size, self.target) else reg } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .memory => |addr| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P")) - .{ .mem = .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = mnem_size.use(op_index) orelse - return self.fail("unknown size: '{s}'", .{op_str}), - .disp = @intCast(@as(i64, @bitCast(addr))), - } }, - } } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .indirect => |reg_off| if (std.mem.eql(u8, modifier, "")) - .{ .mem = .{ - .base = .{ .reg = reg_off.reg }, - .mod = .{ .rm = .{ - .size = mnem_size.use(op_index) orelse - return self.fail("unknown size: '{s}'", .{op_str}), - .disp = reg_off.off, - } }, - } } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .load_frame => |frame_addr| if (std.mem.eql(u8, modifier, "")) - .{ .mem = .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = mnem_size.use(op_index) orelse - return self.fail("unknown size: '{s}'", .{op_str}), - .disp = frame_addr.off, - } }, - } } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .lea_nav => |nav| if (std.mem.eql(u8, modifier, "P")) - .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_nav = nav }) } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .lea_uav => |uav| if (std.mem.eql(u8, modifier, "P")) - .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_uav = uav }) } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .lea_lazy_sym => |lazy_sym| if (std.mem.eql(u8, modifier, "P")) - .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_lazy_sym = lazy_sym }) } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - .lea_extern_func => |extern_func| if (std.mem.eql(u8, modifier, "P")) - .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_extern_func = extern_func }) } - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - else => return self.fail("invalid constraint: '{s}'", .{op_str}), - }; - } else if (std.mem.startsWith(u8, op_str, "$")) { - op.* = if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| - .{ .imm = .u(u) } - else |_| if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| - .{ .imm = .s(s) } - else |_| - return self.fail("invalid immediate: '{s}'", .{op_str}); - } else if (std.mem.endsWith(u8, op_str, ")")) { - const open = std.mem.indexOfScalar(u8, op_str, '(') orelse - return self.fail("invalid operand: '{s}'", .{op_str}); - var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ','); - const base_str = sib_it.next() orelse - return self.fail("invalid memory operand: '{s}'", .{op_str}); - if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%")) - return self.fail("invalid memory operand: '{s}'", .{op_str}); - const index_str = sib_it.next() orelse ""; - if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%")) - return self.fail("invalid memory operand: '{s}'", .{op_str}); - const scale_str = sib_it.next() orelse ""; - if (index_str.len == 0 and scale_str.len > 0) - return self.fail("invalid memory operand: '{s}'", .{op_str}); - const scale: Memory.Scale = if (scale_str.len > 0) - switch (std.fmt.parseInt(u4, scale_str, 10) catch - return self.fail("invalid scale: '{s}'", .{op_str})) { - 1 => .@"1", - 2 => .@"2", - 4 => .@"4", - 8 => .@"8", - else => return self.fail("invalid scale: '{s}'", .{op_str}), - } - else - .@"1"; - if (sib_it.next()) |_| return self.fail("invalid memory operand: '{s}'", .{op_str}); - op.* = if (std.mem.eql(u8, base_str, "%%dx") and index_str.len == 0) - .{ .reg = .dx } - else if (std.mem.eql(u8, base_str, "%%rip") and index_str.len == 0 and - Label.isValid(.reference, op_str[0..open])) - op: { - const anon = std.ascii.isDigit(op_str[0]); - const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else open]); - if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) - return self.fail("undefined label: '{s}'", .{op_str}); - if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; - const pending_relocs = &label_gop.value_ptr.pending_relocs; - if (if (anon) - op_str[1] == 'f' or op_str[1] == 'F' - else - !label_gop.found_existing or pending_relocs.items.len > 0) - try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len)); - break :op .{ .mem = .{ .base = .{ .rip_inst = label_gop.value_ptr.target } } }; - } else .{ .mem = .{ - .base = if (base_str.len > 0) - .{ .reg = parseRegName(base_str["%%".len..]) orelse - return self.fail("invalid base register: '{s}'", .{base_str}) } - else - .none, - .mod = .{ .rm = .{ - .size = mnem_size.use(op_index) orelse return self.fail("unknown size: '{s}'", .{op_str}), - .index = if (index_str.len > 0) - parseRegName(index_str["%%".len..]) orelse - return self.fail("invalid index register: '{s}'", .{op_str}) - else - .none, - .scale = scale, - .disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and - std.mem.endsWith(u8, op_str[0..open], "]")) - disp: { - const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':'); - const modifier = if (colon) |colon_pos| - op_str[colon_pos + ":".len .. open - "]".len] - else - ""; - break :disp switch (args.items[ - arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse - return self.fail("no matching constraint: '{s}'", .{op_str}) - ]) { - .immediate => |imm| if (std.mem.eql(u8, modifier, "") or - std.mem.eql(u8, modifier, "c")) - std.math.cast(i32, @as(i64, @bitCast(imm))) orelse - return self.fail("invalid displacement: '{s}'", .{op_str}) - else - return self.fail("invalid modifier: '{s}'", .{modifier}), - else => return self.fail("invalid constraint: '{s}'", .{op_str}), - }; - } else if (open > 0) - std.fmt.parseInt(i32, op_str[0..open], 0) catch - return self.fail("invalid displacement: '{s}'", .{op_str}) - else - 0, - } }, - } }; - } else if (Label.isValid(.reference, op_str)) { - const anon = std.ascii.isDigit(op_str[0]); - const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]); - if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) - return self.fail("undefined label: '{s}'", .{op_str}); - if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; - const pending_relocs = &label_gop.value_ptr.pending_relocs; - if (if (anon) - op_str[1] == 'f' or op_str[1] == 'F' - else - !label_gop.found_existing or pending_relocs.items.len > 0) - try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len)); - op.* = .{ .inst = label_gop.value_ptr.target }; - } else return self.fail("invalid operand: '{s}'", .{op_str}); - ops_len += 1; - } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str}); - - // convert from att syntax to intel syntax - std.mem.reverse(Operand, ops[0..ops_len]); - if (mnem_size.size != .none and !mnem_size.used) { - comptime var max_mnem_len: usize = 0; - inline for (@typeInfo(encoder.Instruction.Mnemonic).@"enum".fields) |mnem| - max_mnem_len = @max(mnem.name.len, max_mnem_len); - var intel_mnem_buf: [max_mnem_len + 1]u8 = undefined; - const intel_mnem_str = std.fmt.bufPrint(&intel_mnem_buf, "{s}{c}", .{ - @tagName(mnem_tag), - @as(u8, switch (mnem_size.size) { - .byte => 'b', - .word => 'w', - .dword => 'd', - .qword => 'q', - .tbyte => 't', - else => unreachable, - }), - }) catch unreachable; - if (std.meta.stringToEnum(encoder.Instruction.Mnemonic, intel_mnem_str)) |intel_mnem_tag| mnem_tag = intel_mnem_tag; - } - const mnem_name = @tagName(mnem_tag); - const mnem_fixed_tag: Mir.Inst.FixedTag = if (prefix == .directive) - .{ ._, .pseudo } - else for (std.enums.values(Mir.Inst.Fixes)) |fixes| { - const fixes_name = @tagName(fixes); - const space_index = std.mem.indexOfScalar(u8, fixes_name, ' '); - const fixes_prefix = if (space_index) |index| - std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..index]).? - else - .none; - if (fixes_prefix != prefix) continue; - const pattern = fixes_name[if (space_index) |index| index + " ".len else 0..]; - const wildcard_index = std.mem.indexOfScalar(u8, pattern, '_').?; - const mnem_prefix = pattern[0..wildcard_index]; - const mnem_suffix = pattern[wildcard_index + "_".len ..]; - if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue; - if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue; - break .{ fixes, std.meta.stringToEnum( - Mir.Inst.Tag, - mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len], - ) orelse continue }; - } else { - assert(prefix != .none); // no combination of fixes produced a known mnemonic - return self.fail("invalid prefix for mnemonic: '{s} {s}'", .{ - @tagName(prefix), mnem_name, - }); - }; - - (if (prefix == .directive) switch (mnem_tag) { - .@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) - self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm) - else - error.InvalidInstruction, - .@".cfi_def_cfa_register" => if (ops[0] == .reg and ops[1] == .none) - self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, ops[0].reg) - else - error.InvalidInstruction, - .@".cfi_def_cfa_offset" => if (ops[0] == .imm and ops[1] == .none) - self.asmPseudoImmediate(.pseudo_cfi_def_cfa_offset_i_s, ops[0].imm) - else - error.InvalidInstruction, - .@".cfi_adjust_cfa_offset" => if (ops[0] == .imm and ops[1] == .none) - self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, ops[0].imm) - else - error.InvalidInstruction, - .@".cfi_offset" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) - self.asmPseudoRegisterImmediate(.pseudo_cfi_offset_ri_s, ops[0].reg, ops[1].imm) - else - error.InvalidInstruction, - .@".cfi_val_offset" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) - self.asmPseudoRegisterImmediate(.pseudo_cfi_val_offset_ri_s, ops[0].reg, ops[1].imm) - else - error.InvalidInstruction, - .@".cfi_rel_offset" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) - self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, ops[0].reg, ops[1].imm) - else - error.InvalidInstruction, - .@".cfi_register" => if (ops[0] == .reg and ops[1] == .reg and ops[2] == .none) - self.asmPseudoRegisterRegister(.pseudo_cfi_register_rr, ops[0].reg, ops[1].reg) - else - error.InvalidInstruction, - .@".cfi_restore" => if (ops[0] == .reg and ops[1] == .none) - self.asmPseudoRegister(.pseudo_cfi_restore_r, ops[0].reg) - else - error.InvalidInstruction, - .@".cfi_undefined" => if (ops[0] == .reg and ops[1] == .none) - self.asmPseudoRegister(.pseudo_cfi_undefined_r, ops[0].reg) - else - error.InvalidInstruction, - .@".cfi_same_value" => if (ops[0] == .reg and ops[1] == .none) - self.asmPseudoRegister(.pseudo_cfi_same_value_r, ops[0].reg) - else - error.InvalidInstruction, - .@".cfi_remember_state" => if (ops[0] == .none) - self.asmPseudo(.pseudo_cfi_remember_state_none) - else - error.InvalidInstruction, - .@".cfi_restore_state" => if (ops[0] == .none) - self.asmPseudo(.pseudo_cfi_restore_state_none) - else - error.InvalidInstruction, - .@".cfi_escape" => error.InvalidInstruction, - else => unreachable, - } else self.asmOps(mnem_fixed_tag, ops)) catch |err| switch (err) { - error.InvalidInstruction => return self.fail( - "invalid instruction: '{s} {s} {s} {s} {s}'", - .{ - mnem_str, - @tagName(ops[0]), - @tagName(ops[1]), - @tagName(ops[2]), - @tagName(ops[3]), - }, - ), - else => |e| return e, - }; - } - - var label_it = labels.iterator(); - while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0) - return self.fail("undefined label: '{s}'", .{label.key_ptr.*}); - - for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]); - const constraint = - std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]), 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (output == .none) continue; - if (arg_mcv != .register) continue; - if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue; - try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv, .{}); - } - - simple: { - var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); - var buf_index: usize = 0; - for (outputs) |output| { - if (output == .none) continue; - - if (buf_index >= buf.len) break :simple; - buf[buf_index] = output; - buf_index += 1; - } - if (buf_index + inputs.len > buf.len) break :simple; - @memcpy(buf[buf_index..][0..inputs.len], inputs); - return self.finishAir(inst, result, buf); - } - var bt = self.liveness.iterateBigTomb(inst); - for (outputs) |output| if (output != .none) try self.feed(&bt, output); - for (inputs) |input| try self.feed(&bt, input); - return self.finishAirResult(inst, result); -} - -const MoveStrategy = union(enum) { - move_through_gpr, - load_store_through_frame, - load_store: Mir.Inst.FixedTag, - load_store_x87, - insert_extract: InsertExtract, - insert_or_extract_through_gpr: InsertExtract, - vex_insert_extract: InsertExtract, - - const InsertExtract = struct { - insert: Mir.Inst.FixedTag, - extract: Mir.Inst.FixedTag, - }; - - pub fn read(strat: MoveStrategy, cg: *CodeGen, dst_reg: Register, src_mem: Memory) !void { - switch (strat) { - .move_through_gpr => { - const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); - try cg.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), src_mem); - try cg.asmRegisterRegister(.{ ._d, .mov }, dst_reg, tmp_reg.to32()); - }, - .load_store_through_frame => { - const tmp_frame_index = try cg.allocFrameIndex(.init(.{ - .size = 16, - .alignment = .@"16", - })); - const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); - try cg.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), src_mem); - try cg.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = tmp_frame_index }, - .mod = .{ .rm = .{ .size = .dword } }, - }, tmp_reg.to32()); - try cg.asmRegisterMemory(.{ ._ps, .mova }, dst_reg, .{ - .base = .{ .frame = tmp_frame_index }, - .mod = .{ .rm = .{ .size = .xword } }, - }); - }, - .load_store => |tag| { - if (tag[0] == ._ps and tag[1] == .movl) try cg.asmRegisterRegister(.{ ._ps, .xor }, dst_reg, dst_reg); - try cg.asmRegisterMemory(tag, switch (tag[1]) { - else => dst_reg, - .lea => if (dst_reg.size().bitSize(cg.target) >= 32) dst_reg else dst_reg.to32(), - }, src_mem); - }, - .load_store_x87 => if (dst_reg != .st0 and cg.register_manager.isKnownRegFree(.st7)) { - try cg.asmMemory(.{ .f_, .ld }, src_mem); - switch (dst_reg) { - .st1, .st2, .st3, .st4, .st5, .st6 => try cg.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1)), - .st7 => try cg.asmOpOnly(.{ .f_cstp, .in }), - else => unreachable, - } - } else { - try cg.asmRegister(.{ .f_p, .st }, dst_reg); - try cg.asmMemory(.{ .f_, .ld }, src_mem); - switch (dst_reg) { - .st0 => {}, - .st1, .st2, .st3, .st4, .st5, .st6, .st7 => try cg.asmRegister(.{ .f_, .xch }, dst_reg), - else => unreachable, - } - }, - .insert_extract, .insert_or_extract_through_gpr => |ie| { - try cg.asmRegisterRegister(.{ .p_, .xor }, dst_reg, dst_reg); - try cg.asmRegisterMemoryImmediate(ie.insert, dst_reg, src_mem, .u(0)); - }, - .vex_insert_extract => |ie| { - try cg.asmRegisterRegisterRegister(.{ .vp_, .xor }, dst_reg, dst_reg, dst_reg); - try cg.asmRegisterRegisterMemoryImmediate(ie.insert, dst_reg, dst_reg, src_mem, .u(0)); - }, - } - } - pub fn write(strat: MoveStrategy, cg: *CodeGen, dst_mem: Memory, src_reg: Register) !void { - switch (strat) { - .move_through_gpr => { - const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); - try cg.asmRegisterRegister(.{ ._d, .mov }, tmp_reg.to32(), src_reg.to128()); - try cg.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.toSize(dst_mem.mod.rm.size, cg.target)); - }, - .load_store_through_frame => { - const tmp_frame_index = try cg.allocFrameIndex(.init(.{ - .size = 16, - .alignment = .@"16", - })); - try cg.asmMemoryRegister(.{ ._ps, .mova }, .{ - .base = .{ .frame = tmp_frame_index }, - .mod = .{ .rm = .{ .size = .xword } }, - }, src_reg.to128()); - const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); - try cg.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), .{ - .base = .{ .frame = tmp_frame_index }, - .mod = .{ .rm = .{ .size = .word } }, - }); - try cg.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.toSize(dst_mem.mod.rm.size, cg.target)); - }, - .load_store => |tag| try cg.asmMemoryRegister(tag, dst_mem, src_reg), - .load_store_x87 => if (cg.register_manager.isKnownRegFree(.st7)) { - try cg.asmRegister(.{ .f_, .ld }, src_reg); - try cg.asmMemory(.{ .f_p, .st }, dst_mem); - } else { - switch (src_reg) { - .st0 => {}, - .st1, .st2, .st3, .st4, .st5, .st6, .st7 => try cg.asmRegister(.{ .f_, .xch }, src_reg), - else => unreachable, - } - try cg.asmMemory(.{ .f_p, .st }, dst_mem); - try cg.asmMemory(.{ .f_, .ld }, dst_mem); - switch (src_reg) { - .st0 => {}, - .st1, .st2, .st3, .st4, .st5, .st6, .st7 => try cg.asmRegister(.{ .f_, .xch }, src_reg), - else => unreachable, - } - }, - .insert_extract, .vex_insert_extract => |ie| try cg.asmMemoryRegisterImmediate(ie.extract, dst_mem, src_reg, .u(0)), - .insert_or_extract_through_gpr => |ie| { - const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); - try cg.asmRegisterRegisterImmediate(ie.extract, tmp_reg.to32(), src_reg.to128(), .u(0)); - try cg.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16()); - }, - } - } -}; -fn moveStrategy(cg: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy { - const pt = cg.pt; - const zcu = pt.zcu; - switch (class) { - .general_purpose, .gphi, .segment => return .{ .load_store = .{ ._, .mov } }, - .x87 => return .load_store_x87, - .mmx => {}, - .sse => switch (ty.zigTypeTag(zcu)) { - else => { - const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .other), .none); - assert(std.mem.indexOfNone(abi.Class, classes, &.{ - .integer, .sse, .sseup, .memory, .float, .float_combine, - }) == null); - const abi_size = ty.abiSize(zcu); - if (abi_size < 4 or std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) { - 1 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ - .insert = .{ .vp_b, .insr }, - .extract = .{ .vp_b, .extr }, - } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ - .insert = .{ .p_b, .insr }, - .extract = .{ .p_b, .extr }, - } } else if (cg.hasFeature(.sse2)) .move_through_gpr else .load_store_through_frame, - 2 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ - .insert = .{ .vp_w, .insr }, - .extract = .{ .vp_w, .extr }, - } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ - .insert = .{ .p_w, .insr }, - .extract = .{ .p_w, .extr }, - } } else if (cg.hasFeature(.sse2)) .{ .insert_or_extract_through_gpr = .{ - .insert = .{ .p_w, .insr }, - .extract = .{ .p_w, .extr }, - } } else .load_store_through_frame, - 3...4 => return .{ .load_store = .{ if (cg.hasFeature(.avx)) - .v_d - else if (cg.hasFeature(.sse2)) - ._d - else - ._ss, .mov } }, - 5...8 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_q, .mov } - else if (cg.hasFeature(.sse2)) - .{ ._q, .mov } - else - .{ ._ps, .movl } }, - 9...16 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 17...32 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - } else switch (abi_size) { - 4 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_ss, .mov } - else - .{ ._ss, .mov } }, - 5...8 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_sd, .mov } - else if (cg.hasFeature(.sse2)) - .{ ._sd, .mov } - else - .{ ._ps, .movl } }, - 9...16 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_pd, if (aligned) .mova else .movu } - else if (cg.hasFeature(.sse2)) - .{ ._pd, if (aligned) .mova else .movu } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 17...32 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ .v_pd, if (aligned) .mova else .movu } }, - else => {}, - } - }, - .float => switch (ty.floatBits(cg.target)) { - 16 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ - .insert = .{ .vp_w, .insr }, - .extract = .{ .vp_w, .extr }, - } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ - .insert = .{ .p_w, .insr }, - .extract = .{ .p_w, .extr }, - } } else if (cg.hasFeature(.sse2)) .{ .insert_or_extract_through_gpr = .{ - .insert = .{ .p_w, .insr }, - .extract = .{ .p_w, .extr }, - } } else .load_store_through_frame, - 32 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_ss, .mov } - else - .{ ._ss, .mov } }, - 64 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_sd, .mov } - else if (cg.hasFeature(.sse2)) - .{ ._sd, .mov } - else - .{ ._ps, .movl } }, - 128 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - else => {}, - }, - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - .bool => switch (ty.vectorLen(zcu)) { - 1...8 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ - .insert = .{ .vp_b, .insr }, - .extract = .{ .vp_b, .extr }, - } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ - .insert = .{ .p_b, .insr }, - .extract = .{ .p_b, .extr }, - } } else if (cg.hasFeature(.sse2)) .move_through_gpr else .load_store_through_frame, - 9...16 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ - .insert = .{ .vp_w, .insr }, - .extract = .{ .vp_w, .extr }, - } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ - .insert = .{ .p_w, .insr }, - .extract = .{ .p_w, .extr }, - } } else if (cg.hasFeature(.sse2)) .{ .insert_or_extract_through_gpr = .{ - .insert = .{ .p_w, .insr }, - .extract = .{ .p_w, .extr }, - } } else .load_store_through_frame, - 17...32 => return .{ .load_store = .{ if (cg.hasFeature(.avx)) - .v_d - else if (cg.hasFeature(.sse2)) - ._d - else - ._ss, .mov } }, - 33...64 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_q, .mov } - else if (cg.hasFeature(.sse2)) - .{ ._q, .mov } - else - .{ ._ps, .movl } }, - 65...128 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 129...256 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - .int => switch (ty.childType(zcu).intInfo(zcu).bits) { - 1...8 => switch (ty.vectorLen(zcu)) { - 1...16 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 17...32 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - 9...16 => switch (ty.vectorLen(zcu)) { - 1...8 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 9...16 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - 17...32 => switch (ty.vectorLen(zcu)) { - 1...4 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 5...8 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - 33...64 => switch (ty.vectorLen(zcu)) { - 1...2 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 3...4 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - 65...128 => switch (ty.vectorLen(zcu)) { - 1 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 2 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - 129...256 => switch (ty.vectorLen(zcu)) { - 1 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - else => {}, - }, - .pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu)) - switch (ty.vectorLen(zcu)) { - 1...2 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 3...4 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - } - else - unreachable, - .float => switch (ty.childType(zcu).floatBits(cg.target)) { - 16 => switch (ty.vectorLen(zcu)) { - 1...8 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 9...16 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - 32 => switch (ty.vectorLen(zcu)) { - 1...4 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_ps, if (aligned) .mova else .movu } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 5...8 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ .v_ps, if (aligned) .mova else .movu } }, - else => {}, - }, - 64 => switch (ty.vectorLen(zcu)) { - 1...2 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ .v_pd, if (aligned) .mova else .movu } - else if (cg.hasFeature(.sse2)) - .{ ._pd, if (aligned) .mova else .movu } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 3...4 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ .v_pd, if (aligned) .mova else .movu } }, - else => {}, - }, - 80, 128 => switch (ty.vectorLen(zcu)) { - 1 => return .{ .load_store = if (cg.hasFeature(.avx)) - .{ if (aligned) .v_dqa else .v_dqu, .mov } - else if (cg.hasFeature(.sse2)) - .{ if (aligned) ._dqa else ._dqu, .mov } - else - .{ ._ps, if (aligned) .mova else .movu } }, - 2 => if (cg.hasFeature(.avx)) - return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, - else => {}, - }, - else => {}, - }, - else => {}, - }, - }, - .ip, .cr, .dr => {}, - } - return cg.fail("TODO moveStrategy for {f}", .{ty.fmt(pt)}); -} - -const CopyOptions = struct { - safety: bool = false, -}; - -fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void { - const pt = self.pt; - - const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - switch (dst_mcv) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register_overflow, - .register_mask, - .indirect_load_frame, - .lea_frame, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // unmodifiable destination - .register => |dst_reg| try self.genSetReg(dst_reg, ty, src_mcv, opts), - .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) { - .none, - .unreach, - .dead, - .undef, - .register_overflow, - .elementwise_args, - .reserved_frame, - => unreachable, - .immediate, - .register, - .register_offset, - .lea_frame, - => src_mcv.offset(-dst_reg_off.off), - else => .{ .register_offset = .{ - .reg = try self.copyToTmpRegister(ty, src_mcv), - .off = -dst_reg_off.off, - } }, - }, opts), - inline .register_pair, .register_triple, .register_quadruple => |dst_regs, dst_tag| { - const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = src_info: switch (src_mcv) { - .undef, .memory, .indirect, .load_frame => null, - .register => |src_reg| switch (dst_regs[0].class()) { - .general_purpose => switch (src_reg.class()) { - else => unreachable, - .sse => if (ty.abiSize(pt.zcu) <= 16) { - if (self.hasFeature(.avx)) { - try self.asmRegisterRegister(.{ .v_q, .mov }, dst_regs[0].to64(), src_reg.to128()); - try self.asmRegisterRegisterImmediate(.{ .vp_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1)); - } else if (self.hasFeature(.sse4_1)) { - try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128()); - try self.asmRegisterRegisterImmediate(.{ .p_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1)); - } else if (self.hasFeature(.sse2)) { - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128()); - try self.asmRegisterRegister(.{ ._ps, .movhl }, tmp_reg.to128(), src_reg.to128()); - try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[1].to64(), tmp_reg.to128()); - } else { - const frame_index = try self.allocFrameIndex(.init(.{ - .size = 16, - .alignment = .@"16", - })); - try self.asmMemoryRegister(.{ ._ps, .mova }, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .xword } }, - }, src_reg.to128()); - for (dst_regs, 0..) |dst_reg, dst_index| try self.asmRegisterMemory(.{ ._, .mov }, dst_reg.to64(), .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .qword, .disp = @intCast(8 * dst_index) } }, - }); - } - return; - } else unreachable, - }, - else => unreachable, - }, - dst_tag => |src_regs| { - var remaining: std.StaticBitSet(dst_regs.len) = .initFull(); - var hazard_regs = src_regs; - while (!remaining.eql(.initEmpty())) { - var remaining_it = remaining.iterator(.{}); - next: while (remaining_it.next()) |index| { - const dst_reg = dst_regs[index]; - const dst_id = dst_reg.id(); - const src_reg = hazard_regs[index]; - const src_id = src_reg.id(); - if (dst_id == src_id) { - remaining.unset(index); - continue; - } - var mir_tag: Mir.Inst.FixedTag = .{ ._, .mov }; - var hazard_it = remaining.iterator(.{}); - while (hazard_it.next()) |hazard_index| { - if (dst_id != hazard_regs[hazard_index].id()) continue; - for (dst_regs) |clobber_reg| { - if (src_id == clobber_reg.id()) break; - } else continue :next; - hazard_regs[hazard_index] = src_reg; - mir_tag = .{ ._g, .xch }; - } - remaining.unset(index); - try self.asmRegisterRegister(mir_tag, dst_reg.to64(), src_reg.to64()); - } - } - return; - }, - .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => { - const src_addr_reg = - (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); - const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); - errdefer self.register_manager.unlockReg(src_addr_lock); - - try self.genSetReg(src_addr_reg, .usize, src_mcv.address(), opts); - break :src_info .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; - }, - .air_ref => |src_ref| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref), opts), - else => return self.fail("TODO implement genCopy for {s} of {f}", .{ - @tagName(src_mcv), ty.fmt(pt), - }), - }; - defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); - - for ([_]bool{ false, true }) |emit_hazard| { - var hazard_count: u3 = 0; - var part_disp: i32 = 0; - for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| { - defer part_disp += @intCast(dst_ty.abiSize(pt.zcu)); - const is_hazard = if (src_mcv.getReg()) |src_reg| - dst_reg.id() == src_reg.id() - else if (src_info) |info| - dst_reg.id() == info.addr_reg.id() - else - false; - if (is_hazard) hazard_count += 1; - if (is_hazard != emit_hazard) continue; - try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) { - .undef => if (opts.safety and part_i > 0) .{ .register = dst_regs[0] } else .undef, - dst_tag => |src_regs| .{ .register = src_regs[part_i] }, - .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), - .load_nav, - .load_uav, - .load_lazy_sym, - .load_extern_func, - => .{ .indirect = .{ - .reg = src_info.?.addr_reg, - .off = part_disp, - } }, - else => unreachable, - }, opts); - } - switch (hazard_count) { - 0 => break, - 1 => continue, - else => unreachable, - } - } - }, - .indirect => |reg_off| try self.genSetMem( - .{ .reg = reg_off.reg }, - reg_off.off, - ty, - src_mcv, - opts, - ), - .memory => { - switch (dst_mcv) { - .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| - return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts), - .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, - else => unreachable, - } - - const addr_reg = try self.copyToTmpRegister(.usize, dst_mcv.address()); - const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_lock); - - try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv, opts); - }, - .load_frame => |frame_addr| try self.genSetMem( - .{ .frame = frame_addr.index }, - frame_addr.off, - ty, - src_mcv, - opts, - ), - .load_nav => |nav| try self.genSetMem(.{ .nav = nav }, 0, ty, src_mcv, opts), - .load_uav => |uav| try self.genSetMem(.{ .uav = uav }, 0, ty, src_mcv, opts), - .load_lazy_sym => |lazy_sym| try self.genSetMem(.{ .lazy_sym = lazy_sym }, 0, ty, src_mcv, opts), - .load_extern_func => |extern_func| try self.genSetMem(.{ .extern_func = extern_func }, 0, ty, src_mcv, opts), - } -} - -fn genSetReg( - self: *CodeGen, - dst_reg: Register, - ty: Type, - src_mcv: MCValue, - opts: CopyOptions, -) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const dst_alias = registerAlias(dst_reg, abi_size); - if (ty.bitSize(zcu) > dst_alias.size().bitSize(self.target)) - return self.fail("genSetReg called with a value larger than dst_reg", .{}); - switch (src_mcv) { - .none, - .unreach, - .dead, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - => unreachable, - .undef => if (opts.safety) switch (dst_reg.class()) { - .general_purpose, .gphi => switch (abi_size) { - 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xaa)), - 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xaaaa)), - 3...4 => try self.asmRegisterImmediate( - .{ ._, .mov }, - dst_reg.to32(), - .s(@as(i32, @bitCast(@as(u32, 0xaaaaaaaa)))), - ), - 5...8 => try self.asmRegisterImmediate( - .{ ._, .mov }, - dst_reg.to64(), - .u(0xaaaaaaaaaaaaaaaa), - ), - else => unreachable, - }, - .segment, .mmx, .sse => { - const full_ty = try pt.vectorType(.{ - .len = self.vectorSize(.float), - .child = .u8_type, - }); - try self.genSetReg(dst_reg, full_ty, try self.lowerValue( - try pt.aggregateSplatValue(full_ty, try pt.intValue(.u8, 0xaa)), - ), opts); - }, - .x87 => try self.genSetReg(dst_reg, .f80, try self.lowerValue( - try pt.floatValue(.f80, @as(f80, @bitCast(@as(u80, 0xaaaaaaaaaaaaaaaaaaaa)))), - ), opts), - .ip, .cr, .dr => unreachable, - }, - .eflags => |cc| try self.asmSetccRegister(cc, dst_reg.to8()), - .immediate => |imm| { - if (imm == 0) { - // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit - // register is the fastest way to zero a register. - try self.spillEflagsIfOccupied(); - try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); - } else if (abi_size > 4 and std.math.cast(u32, imm) != null) { - // 32-bit moves zero-extend to 64-bit. - try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm)); - } else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) { - try self.asmRegisterImmediate( - .{ ._, .mov }, - dst_alias, - .s(@intCast(@as(i64, @bitCast(imm)))), - ); - } else { - try self.asmRegisterImmediate( - .{ ._, .mov }, - dst_alias, - .u(imm), - ); - } - }, - .register => |src_reg| if (dst_reg.id() != src_reg.id()) switch (dst_reg.class()) { - .general_purpose => switch (src_reg.class()) { - .general_purpose => try self.asmRegisterRegister( - .{ ._, .mov }, - dst_alias, - registerAlias(src_reg, abi_size), - ), - .gphi => if (dst_reg.isClass(.gphi)) try self.asmRegisterRegister( - .{ ._, .mov }, - dst_alias, - registerAlias(src_reg, abi_size), - ) else { - const src_lock = self.register_manager.lockReg(src_reg); - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gphi); - - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg.to8(), src_reg); - try self.asmRegisterRegister(.{ ._, .mov }, dst_alias, tmp_reg.to8()); - }, - .segment => try self.asmRegisterRegister( - .{ ._, .mov }, - dst_alias, - src_reg, - ), - .x87, .mmx, .ip, .cr, .dr => unreachable, - .sse => if (self.hasFeature(.sse2)) try self.asmRegisterRegister( - switch (abi_size) { - 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, - 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, - else => unreachable, - }, - registerAlias(dst_reg, @max(abi_size, 4)), - src_reg.to128(), - ) else { - const frame_size = std.math.ceilPowerOfTwoAssert(u32, @max(abi_size, 4)); - const frame_index = try self.allocFrameIndex(.init(.{ - .size = frame_size, - .alignment = .fromNonzeroByteUnits(frame_size), - })); - try self.asmMemoryRegister(switch (frame_size) { - 4 => .{ ._ss, .mov }, - 8 => .{ ._ps, .movl }, - 16 => .{ ._ps, .mov }, - else => unreachable, - }, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, - }, src_reg.to128()); - try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, - }); - }, - }, - .gphi => switch (src_reg.class()) { - .general_purpose => if (src_reg.isClass(.gphi)) try self.asmRegisterRegister( - .{ ._, .mov }, - dst_alias, - registerAlias(src_reg, abi_size), - ) else { - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gphi); - - try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg.to8(), src_reg.to8()); - try self.asmRegisterRegister(.{ ._, .mov }, dst_reg, tmp_reg.to8()); - }, - .gphi => try self.asmRegisterRegister( - .{ ._, .mov }, - dst_alias, - registerAlias(src_reg, abi_size), - ), - .segment, .x87, .mmx, .ip, .cr, .dr, .sse => unreachable, - }, - .segment => try self.asmRegisterRegister( - .{ ._, .mov }, - dst_reg, - switch (src_reg.class()) { - .general_purpose, .gphi, .segment => registerAlias(src_reg, abi_size), - .x87, .mmx, .ip, .cr, .dr => unreachable, - .sse => try self.copyToTmpRegister(ty, src_mcv), - }, - ), - .x87 => switch (src_reg.class()) { - .general_purpose, .gphi, .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87 => switch (src_reg) { - .st0 => try self.asmRegister(.{ .f_, .st }, dst_reg), - .st1, .st2, .st3, .st4, .st5, .st6 => switch (dst_reg) { - .st0 => { - try self.asmRegister(.{ .f_p, .st }, .st0); - try self.asmRegister(.{ .f_, .ld }, @enumFromInt(@intFromEnum(src_reg) - 1)); - }, - .st2, .st3, .st4, .st5, .st6 => if (self.register_manager.isKnownRegFree(.st7)) { - try self.asmRegister(.{ .f_, .ld }, src_reg); - try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1)); - } else { - try self.asmRegister(.{ .f_, .xch }, src_reg); - try self.asmRegister(.{ .f_, .xch }, dst_reg); - try self.asmRegister(.{ .f_, .xch }, src_reg); - }, - .st7 => { - if (!self.register_manager.isKnownRegFree(.st7)) try self.asmRegister(.{ .f_, .free }, dst_reg); - try self.asmRegister(.{ .f_, .ld }, src_reg); - try self.asmOpOnly(.{ .f_cstp, .in }); - }, - else => unreachable, - }, - else => unreachable, - }, - .sse => if (abi_size <= 16) { - const frame_index = try self.allocFrameIndex(.init(.{ - .size = 16, - .alignment = .@"16", - })); - try self.asmMemoryRegister(if (self.hasFeature(.avx)) - .{ .v_dqa, .mov } - else if (self.hasFeature(.sse2)) - .{ ._dqa, .mov } - else - .{ ._ps, .mova }, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .xword } }, - }, src_reg.to128()); - try MoveStrategy.read(.load_store_x87, self, dst_reg, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .tbyte } }, - }); - } else unreachable, - }, - .mmx => unreachable, - .sse => switch (src_reg.class()) { - .general_purpose, .gphi => if (self.hasFeature(.sse2)) try self.asmRegisterRegister( - switch (abi_size) { - 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, - 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, - else => unreachable, - }, - dst_reg.to128(), - registerAlias(src_reg, @max(abi_size, 4)), - ) else { - const frame_size = std.math.ceilPowerOfTwoAssert(u32, @max(abi_size, 4)); - const frame_index = try self.allocFrameIndex(.init(.{ - .size = frame_size, - .alignment = .fromNonzeroByteUnits(frame_size), - })); - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, - }, registerAlias(src_reg, abi_size)); - switch (frame_size) { - else => {}, - 8 => try self.asmRegisterRegister(.{ ._ps, .xor }, dst_reg.to128(), dst_reg.to128()), - } - try self.asmRegisterMemory(switch (frame_size) { - 4 => .{ ._ss, .mov }, - 8 => .{ ._ps, .movl }, - 16 => .{ ._ps, .mova }, - else => unreachable, - }, dst_reg.to128(), .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, - }); - }, - .segment => try self.genSetReg( - dst_reg, - ty, - .{ .register = try self.copyToTmpRegister(ty, src_mcv) }, - opts, - ), - .x87 => if (abi_size <= 16) { - const frame_index = try self.allocFrameIndex(.init(.{ - .size = 16, - .alignment = .@"16", - })); - try MoveStrategy.write(.load_store_x87, self, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .tbyte } }, - }, src_reg); - try self.asmRegisterMemory(if (self.hasFeature(.avx)) - .{ .v_dqa, .mov } - else if (self.hasFeature(.sse2)) - .{ ._dqa, .mov } - else - .{ ._ps, .mova }, dst_reg.to128(), .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .xword } }, - }); - } else unreachable, - .mmx, .ip, .cr, .dr => unreachable, - .sse => try self.asmRegisterRegister( - @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) { - else => switch (abi_size) { - 1...16 => if (self.hasFeature(.avx)) - .{ .v_dqa, .mov } - else if (self.hasFeature(.sse2)) - .{ ._dqa, .mov } - else - .{ ._ps, .mova }, - 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, - else => null, - }, - .float => switch (ty.scalarType(zcu).floatBits(self.target)) { - 16, 128 => switch (abi_size) { - 2...16 => if (self.hasFeature(.avx)) - .{ .v_dqa, .mov } - else if (self.hasFeature(.sse2)) - .{ ._dqa, .mov } - else - .{ ._ps, .mova }, - 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, - else => null, - }, - 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova }, - 64 => if (self.hasFeature(.avx)) - .{ .v_pd, .mova } - else if (self.hasFeature(.sse2)) - .{ ._pd, .mova } - else - .{ ._ps, .mova }, - 80 => null, - else => unreachable, - }, - }) orelse return self.fail("TODO implement genSetReg for {f}", .{ty.fmt(pt)}), - dst_alias, - registerAlias(src_reg, abi_size), - ), - }, - .ip, .cr, .dr => unreachable, - } else if ((dst_reg.class() == .gphi) != (src_reg.class() == .gphi)) try self.asmRegisterRegister( - .{ ._, .mov }, - dst_reg.to8(), - src_reg.to8(), - ), - inline .register_pair, - .register_triple, - .register_quadruple, - => |src_regs| switch (dst_reg.class()) { - .general_purpose => switch (src_regs[0].class()) { - .general_purpose => try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts), - else => unreachable, - }, - .sse => switch (src_regs[0].class()) { - .general_purpose => if (abi_size <= 16) { - if (self.hasFeature(.avx)) { - try self.asmRegisterRegister(.{ .v_q, .mov }, dst_reg.to128(), src_regs[0].to64()); - try self.asmRegisterRegisterRegisterImmediate( - .{ .vp_q, .insr }, - dst_reg.to128(), - dst_reg.to128(), - src_regs[1].to64(), - .u(1), - ); - } else if (self.hasFeature(.sse4_1)) { - try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64()); - try self.asmRegisterRegisterImmediate(.{ .p_q, .insr }, dst_reg.to128(), src_regs[1].to64(), .u(1)); - } else if (self.hasFeature(.sse2)) { - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64()); - try self.asmRegisterRegister(.{ ._q, .mov }, tmp_reg.to128(), src_regs[1].to64()); - try self.asmRegisterRegister(.{ ._ps, .movlh }, dst_reg.to128(), tmp_reg.to128()); - } else { - const frame_index = try self.allocFrameIndex(.init(.{ - .size = 16, - .alignment = .@"16", - })); - for (src_regs, 0..) |src_reg, src_index| try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .qword, .disp = @intCast(8 * src_index) } }, - }, src_reg.to64()); - try self.asmRegisterMemory(.{ ._ps, .mova }, dst_reg.to128(), .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .xword } }, - }); - } - } else unreachable, - else => unreachable, - }, - else => unreachable, - }, - .register_offset, - .indirect, - .load_frame, - .lea_frame, - => try @as(MoveStrategy, switch (src_mcv) { - .register_offset => |reg_off| switch (reg_off.off) { - 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }, opts), - else => .{ .load_store = .{ ._, .lea } }, - }, - .indirect => try self.moveStrategy(ty, dst_reg.class(), false), - .load_frame => |frame_addr| try self.moveStrategy( - ty, - dst_reg.class(), - self.getFrameAddrAlignment(frame_addr).compare(.gte, .fromLog2Units( - std.math.log2_int_ceil(u64, @divExact(dst_reg.size().bitSize(self.target), 8)), - )), - ), - .lea_frame => .{ .load_store = .{ ._, .lea } }, - else => unreachable, - }).read(self, dst_alias, switch (src_mcv) { - .register_offset, .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg.to64() }, - .mod = .{ .rm = .{ - .size = self.memSize(ty), - .disp = reg_off.off, - } }, - }, - .load_frame, .lea_frame => |frame_addr| .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = self.memSize(ty), - .disp = frame_addr.off, - } }, - }, - else => unreachable, - }), - .register_overflow => |src_reg_ov| { - const ip = &zcu.intern_pool; - const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(ty.toIntern())) { - .tuple_type => |tuple_type| { - const tuple_field_types = tuple_type.types.get(ip); - assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type); - break :first_ty tuple_field_types[0]; - }, - .opt_type => |opt_child| { - assert(!ty.optionalReprIsPayload(zcu)); - break :first_ty opt_child; - }, - else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, ty.fmt(pt) }), - }); - const first_size: u31 = @intCast(first_ty.abiSize(zcu)); - const frame_size = std.math.ceilPowerOfTwoAssert(u32, abi_size); - const frame_index = try self.allocFrameIndex(.init(.{ - .size = frame_size, - .alignment = .fromNonzeroByteUnits(frame_size), - })); - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(first_size) } }, - }, registerAlias(src_reg_ov.reg, first_size)); - try self.asmSetccMemory(src_reg_ov.eflags, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .byte, .disp = first_size } }, - }); - try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, - }); - }, - .register_mask => |src_reg_mask| { - assert(src_reg_mask.reg.isClass(.sse)); - const has_avx = self.hasFeature(.avx); - const bits_reg = switch (dst_reg.class()) { - .general_purpose => dst_reg, - else => try self.register_manager.allocReg(null, abi.RegisterClass.gp), - }; - const bits_lock = self.register_manager.lockReg(bits_reg); - defer if (bits_lock) |lock| self.register_manager.unlockReg(lock); - - const pack_reg = switch (src_reg_mask.info.scalar) { - else => src_reg_mask.reg, - .word => try self.register_manager.allocReg(null, abi.RegisterClass.sse), - }; - const pack_lock = self.register_manager.lockReg(pack_reg); - defer if (pack_lock) |lock| self.register_manager.unlockReg(lock); - - var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(self.target), 8)); - switch (src_reg_mask.info.scalar) { - else => {}, - .word => { - const src_alias = registerAlias(src_reg_mask.reg, mask_size); - const pack_alias = registerAlias(pack_reg, mask_size); - if (has_avx) { - try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias); - } else { - try self.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias); - try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias); - } - mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable; - }, - } - try self.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) { - .byte, .word => if (has_avx) .vp_b else .p_b, - .dword => if (has_avx) .v_ps else ._ps, - .qword => if (has_avx) .v_pd else ._pd, - else => unreachable, - }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size)); - if (src_reg_mask.info.inverted) try self.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size)); - try self.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{}); - }, - .memory, .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => { - switch (src_mcv) { - .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| - return (try self.moveStrategy( - ty, - dst_reg.class(), - ty.abiAlignment(zcu).check(@as(u32, @bitCast(small_addr))), - )).read(self, dst_alias, .{ - .base = .{ .reg = .ds }, - .mod = .{ .rm = .{ - .size = self.memSize(ty), - .disp = small_addr, - } }, - }), - .load_nav => |nav| switch (dst_reg.class()) { - .general_purpose, .gphi => { - try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ - .base = .{ .nav = nav }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }); - return; - }, - .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87, .sse => {}, - }, - .load_uav => |uav| switch (dst_reg.class()) { - .general_purpose, .gphi => { - try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ - .base = .{ .uav = uav }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }); - return; - }, - .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87, .sse => {}, - }, - .load_lazy_sym => |lazy_sym| switch (dst_reg.class()) { - .general_purpose, .gphi => { - try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ - .base = .{ .lazy_sym = lazy_sym }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }); - return; - }, - .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87, .sse => {}, - }, - .load_extern_func => |extern_func| switch (dst_reg.class()) { - .general_purpose, .gphi => { - try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ - .base = .{ .extern_func = extern_func }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }); - return; - }, - .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87, .sse => {}, - }, - else => unreachable, - } - - const addr_reg = try self.copyToTmpRegister(.usize, src_mcv.address()); - const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); - defer self.register_manager.unlockReg(addr_lock); - - try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, dst_alias, .{ - .base = .{ .reg = addr_reg.to64() }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }); - }, - .lea_nav => |nav| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ - .base = .{ .nav = nav }, - }), - .lea_uav => |uav| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ - .base = .{ .uav = uav }, - }), - .lea_lazy_sym => |lazy_sym| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ - .base = .{ .lazy_sym = lazy_sym }, - }), - .lea_extern_func => |lazy_sym| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ - .base = .{ .extern_func = lazy_sym }, - }), - .air_ref => |src_ref| try self.genSetReg(dst_reg, ty, try self.resolveInst(src_ref), opts), - } -} - -fn genSetMem( - self: *CodeGen, - base: Memory.Base, - disp: i32, - ty: Type, - src_mcv: MCValue, - opts: CopyOptions, -) InnerError!void { - const pt = self.pt; - const zcu = pt.zcu; - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const dst_ptr_mcv: MCValue = switch (base) { - .none => .{ .immediate = @bitCast(@as(i64, disp)) }, - .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, - .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, - .table, .rip_inst, .lazy_sym => unreachable, - .nav => |nav| .{ .lea_nav = nav }, - .uav => |uav| .{ .lea_uav = uav }, - .extern_func => |extern_func| .{ .lea_extern_func = extern_func }, - }; - switch (src_mcv) { - .none, - .unreach, - .dead, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - => unreachable, - .undef => if (opts.safety) try self.genInlineMemset( - dst_ptr_mcv, - src_mcv, - .{ .immediate = abi_size }, - opts, - ), - .immediate => |imm| switch (abi_size) { - 1, 2, 4 => { - const immediate: Immediate = switch (if (ty.isAbiInt(zcu)) - ty.intInfo(zcu).signedness - else - .unsigned) { - .signed => .s(@truncate(@as(i64, @bitCast(imm)))), - .unsigned => .u(@as(u32, @intCast(imm))), - }; - try self.asmMemoryImmediate( - .{ ._, .mov }, - .{ .base = base, .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = disp, - } } }, - immediate, - ); - }, - 3, 5...7 => unreachable, - else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| { - try self.asmMemoryImmediate( - .{ ._, .mov }, - .{ .base = base, .mod = .{ .rm = .{ - .size = .fromSize(abi_size), - .disp = disp, - } } }, - .s(small), - ); - } else { - var offset: i32 = 0; - while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate( - .{ ._, .mov }, - .{ .base = base, .mod = .{ .rm = .{ - .size = .dword, - .disp = disp + offset, - } } }, - if (ty.isSignedInt(zcu)) .s( - @truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)), - ) else .u( - @as(u32, @truncate(if (std.math.cast(u6, offset * 8)) |shift| imm >> shift else 0)), - ), - ); - }, - }, - .eflags => |cc| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{ - .rm = .{ .size = .byte, .disp = disp }, - } }), - .register => |src_reg| { - const ip = &zcu.intern_pool; - const mem_size = switch (base) { - .frame => |base_fi| mem_size: { - assert(disp >= 0); - const frame_abi_size = self.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)]; - const frame_spill_pad = self.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)]; - assert(frame_abi_size - frame_spill_pad - disp >= abi_size); - break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size) frame_abi_size else abi_size; - }, - else => abi_size, - }; - const src_alias = registerAlias(src_reg, @intCast(self.unalignedSize(ty))); - const src_class = src_alias.class(); - const src_size: Memory.Size = switch (src_class) { - .general_purpose, .gphi, .segment, .ip, .cr, .dr => src_alias.size(), - .mmx, .sse => .fromSize(abi_size), - .x87 => switch (abi.classifySystemV(ty, zcu, self.target, .other)[0]) { - else => unreachable, - .float => .dword, - .float_combine, .sse => .qword, - .x87 => .tbyte, - }, - }; - const src_bit_size = src_size.bitSize(self.target); - const src_align: InternPool.Alignment = .fromNonzeroByteUnits(std.math.ceilPowerOfTwoAssert(u64, src_bit_size)); - const src_byte_size = @divExact(src_bit_size, 8); - if (src_byte_size > mem_size) { - const frame_index = try self.allocFrameIndex(.init(.{ - .size = src_byte_size, - .alignment = src_align, - })); - const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } }; - try (try self.moveStrategy(ty, src_class, true)).write(self, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = src_size } }, - }, src_alias); - try self.genSetMem(base, disp, ty, frame_mcv, opts); - try self.freeValue(frame_mcv, .{}); - } else try (try self.moveStrategy(ty, src_class, switch (base) { - .none => src_align.check(@as(u32, @bitCast(disp))), - .reg => |reg| switch (reg) { - .es, .cs, .ss, .ds => src_align.check(@as(u32, @bitCast(disp))), - else => false, - }, - .frame => |frame_index| self.getFrameAddrAlignment(.{ - .index = frame_index, - .off = disp, - }).compare(.gte, src_align), - .table, .rip_inst, .lazy_sym, .extern_func => unreachable, - .nav => |nav| ip.getNav(nav).getAlignment().compare(.gte, src_align), - .uav => |uav| Type.fromInterned(uav.orig_ty).ptrAlignment(zcu).compare(.gte, src_align), - })).write(self, .{ - .base = base, - .mod = .{ .rm = .{ .size = src_size, .disp = disp } }, - }, src_alias); - }, - inline .register_pair, .register_triple, .register_quadruple => |src_regs| { - var part_disp: i32 = disp; - for (try self.splitType(src_regs.len, ty), src_regs) |src_ty, src_reg| { - try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts); - part_disp += @intCast(src_ty.abiSize(zcu)); - } - }, - .register_overflow => |ro| switch (ty.zigTypeTag(zcu)) { - .@"struct" => { - try self.genSetMem( - base, - disp + @as(i32, @intCast(ty.structFieldOffset(0, zcu))), - ty.fieldType(0, zcu), - .{ .register = ro.reg }, - opts, - ); - try self.genSetMem( - base, - disp + @as(i32, @intCast(ty.structFieldOffset(1, zcu))), - ty.fieldType(1, zcu), - .{ .eflags = ro.eflags }, - opts, - ); - }, - .optional => { - assert(!ty.optionalReprIsPayload(zcu)); - const child_ty = ty.optionalChild(zcu); - try self.genSetMem(base, disp, child_ty, .{ .register = ro.reg }, opts); - try self.genSetMem( - base, - disp + @as(i32, @intCast(child_ty.abiSize(zcu))), - .bool, - .{ .eflags = ro.eflags }, - opts, - ); - }, - else => return self.fail("TODO implement genSetMem for {s} of {f}", .{ - @tagName(src_mcv), ty.fmt(pt), - }), - }, - .register_offset => |reg_off| { - const src_reg = self.copyToTmpRegister(ty, src_mcv) catch |err| switch (err) { - error.OutOfRegisters => { - const src_reg = registerAlias(reg_off.reg, abi_size); - try self.asmRegisterMemory(.{ ._, .lea }, src_reg, .{ - .base = .{ .reg = src_reg }, - .mod = .{ .rm = .{ .disp = reg_off.off } }, - }); - try self.genSetMem(base, disp, ty, .{ .register = reg_off.reg }, opts); - return self.asmRegisterMemory(.{ ._, .lea }, src_reg, .{ - .base = .{ .reg = src_reg }, - .mod = .{ .rm = .{ .disp = -reg_off.off } }, - }); - }, - else => |e| return e, - }; - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); - }, - .register_mask => { - const src_reg = try self.copyToTmpRegister(ty, src_mcv); - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); - }, - .memory, - .indirect, - .load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => switch (abi_size) { - 0 => {}, - 1, 2, 4, 8 => { - const src_reg = try self.copyToTmpRegister(ty, src_mcv); - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); - }, - else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }, .{ .no_alias = true }), - }, - .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts), - } -} - -fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct { - no_alias: bool, -}) InnerError!void { - if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) { - else => {}, - .immediate => |len_imm| switch (len_imm) { - else => {}, - 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { - try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte })); - try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8()); - return; - }, - 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { - try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word })); - try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16()); - return; - }, - 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { - try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword })); - try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32()); - return; - }, - 8 => if (self.target.cpu.arch == .x86_64) { - if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { - try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword })); - try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64()); - return; - } - }, - 16 => if (self.hasFeature(.avx)) { - if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { - try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); - try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); - return; - } - } else if (self.hasFeature(.sse2)) { - if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { - try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); - try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); - return; - } - } else if (self.hasFeature(.sse)) { - if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { - try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); - try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); - return; - } - }, - 32 => if (self.hasFeature(.avx)) { - if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { - try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword })); - try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256()); - return; - } - }, - }, - }; - try self.spillRegisters(&.{ .rsi, .rdi, .rcx }); - try self.genSetReg(.rsi, .usize, src_ptr, .{}); - try self.genSetReg(.rdi, .usize, dst_ptr, .{}); - try self.genSetReg(.rcx, .usize, len, .{}); - try self.asmOpOnly(.{ .@"rep _sb", .mov }); -} - -fn genInlineMemset( - self: *CodeGen, - dst_ptr: MCValue, - value: MCValue, - len: MCValue, - opts: CopyOptions, -) InnerError!void { - try self.spillRegisters(&.{ .rdi, .al, .rcx }); - try self.genSetReg(.rdi, .usize, dst_ptr, .{}); - try self.genSetReg(.al, .u8, value, opts); - try self.genSetReg(.rcx, .usize, len, .{}); - try self.asmOpOnly(.{ .@"rep _sb", .sto }); -} - -fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const dst_ty = self.typeOfIndex(inst); - const src_ty = self.typeOf(ty_op.operand); - - const result = result: { - const src_mcv = try self.resolveInst(ty_op.operand); - if (dst_ty.isPtrAtRuntime(zcu) and src_ty.isPtrAtRuntime(zcu)) switch (src_mcv) { - .lea_frame => break :result src_mcv, - else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv, - }; - - const dst_rc = self.regSetForType(dst_ty); - const src_rc = self.regSetForType(src_ty); - - const src_lock = if (src_mcv.getReg()) |src_reg| self.register_manager.lockReg(src_reg) else null; - defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = if (src_mcv != .register_mask and - (if (src_mcv.getReg()) |src_reg| src_reg.isClass(.general_purpose) else true) and - dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and - dst_ty.abiAlignment(zcu).order(src_ty.abiAlignment(zcu)).compare(.lte) and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(switch (src_mcv) { - else => switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { - .lt => dst_ty, - .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty, - .gt => src_ty, - }, - .register_mask => src_ty, - }, dst_mcv, src_mcv, .{}); - break :dst dst_mcv; - }; - - if (dst_ty.isRuntimeFloat()) break :result dst_mcv; - - if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and - dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv; - - const abi_size = dst_ty.abiSize(zcu); - const bit_size = dst_ty.bitSize(zcu); - if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv; - - const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable; - const high_mcv: MCValue = switch (dst_mcv) { - .register => |dst_reg| .{ .register = dst_reg }, - .register_pair => |dst_regs| .{ .register = dst_regs[1] }, - else => dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(), - }; - const high_reg = if (high_mcv.isRegister()) - high_mcv.getReg().? - else - try self.copyToTmpRegister(.usize, high_mcv); - const high_lock = self.register_manager.lockReg(high_reg); - defer if (high_lock) |lock| self.register_manager.unlockReg(lock); - try self.truncateRegister(dst_ty, high_reg); - if (!high_mcv.isRegister()) try self.genCopy( - if (abi_size <= 8) dst_ty else .usize, - high_mcv, - .{ .register = high_reg }, - .{}, - ); - var offset = dst_limbs_len * 8; - if (offset < abi_size) { - const dst_signedness: std.builtin.Signedness = if (dst_ty.isAbiInt(zcu)) - dst_ty.intInfo(zcu).signedness - else - .unsigned; - const ext_mcv: MCValue = ext_mcv: switch (dst_signedness) { - .signed => { - try self.asmRegisterImmediate(.{ ._r, .sa }, high_reg, .u(63)); - break :ext_mcv .{ .register = high_reg }; - }, - .unsigned => .{ .immediate = 0 }, - }; - while (offset < abi_size) : (offset += 8) { - const limb_mcv: MCValue = switch (dst_mcv) { - .register => |dst_reg| .{ .register = dst_reg }, - .register_pair => |dst_regs| .{ .register = dst_regs[@divExact(offset, 8)] }, - else => dst_mcv.address().offset(offset).deref(), - }; - const limb_lock = if (limb_mcv.isRegister()) - self.register_manager.lockReg(limb_mcv.getReg().?) - else - null; - defer if (limb_lock) |lock| self.register_manager.unlockReg(lock); - try self.genCopy(.usize, limb_mcv, ext_mcv, .{}); - } - } - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const slice_ty = self.typeOfIndex(inst); - const ptr_ty = self.typeOf(ty_op.operand); - const ptr = try self.resolveInst(ty_op.operand); - const array_ty = ptr_ty.childType(zcu); - const array_len = array_ty.arrayLen(zcu); - - const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu)); - try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{}); - try self.genSetMem( - .{ .frame = frame_index }, - @intCast(ptr_ty.abiSize(zcu)), - .usize, - .{ .immediate = array_len }, - .{}, - ); - - const result = MCValue{ .load_frame = .{ .index = frame_index } }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dst_ty = self.typeOfIndex(inst); - const dst_bits = dst_ty.floatBits(self.target); - - const src_ty = self.typeOf(ty_op.operand); - const src_bits: u32 = @intCast(src_ty.bitSize(zcu)); - const src_signedness = - if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned; - const src_size = std.math.divCeil(u32, @max(switch (src_signedness) { - .signed => src_bits, - .unsigned => src_bits + 1, - }, 32), 8) catch unreachable; - - const result = result: { - if (switch (dst_bits) { - 16, 80, 128 => true, - 32, 64 => src_size > 8, - else => unreachable, - }) { - if (src_bits > 128) return self.fail("TODO implement airFloatFromInt from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - - var sym_buf: ["__floatun?i?f".len]u8 = undefined; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = dst_ty.toIntern(), - .param_types = &.{src_ty.toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "__float{s}{c}i{c}f", .{ - switch (src_signedness) { - .signed => "", - .unsigned => "un", - }, - intCompilerRtAbiName(src_bits), - floatCompilerRtAbiName(dst_bits), - }) catch unreachable, - } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); - } - - const src_mcv = try self.resolveInst(ty_op.operand); - const src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv); - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg); - - const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty)); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_ty.zigTypeTag(zcu)) { - .float => switch (dst_ty.floatBits(self.target)) { - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .cvtsi2 } else .{ ._ss, .cvtsi2 }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .cvtsi2 } else .{ ._sd, .cvtsi2 }, - 16, 80, 128 => null, - else => unreachable, - }, - else => null, - }) orelse return self.fail("TODO implement airFloatFromInt from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - const dst_alias = dst_reg.to128(); - const src_alias = registerAlias(src_reg, src_size); - switch (mir_tag[0]) { - .v_ss, .v_sd => try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, src_alias), - else => try self.asmRegisterRegister(mir_tag, dst_alias, src_alias), - } - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const dst_ty = self.typeOfIndex(inst); - const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu)); - const dst_signedness = - if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned; - const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) { - .signed => dst_bits, - .unsigned => dst_bits + 1, - }, 32), 8) catch unreachable; - - const src_ty = self.typeOf(ty_op.operand); - const src_bits = src_ty.floatBits(self.target); - - const result = result: { - if (switch (src_bits) { - 16, 80, 128 => true, - 32, 64 => dst_size > 8, - else => unreachable, - }) { - if (dst_bits > 128) return self.fail("TODO implement airIntFromFloat from {f} to {f}", .{ - src_ty.fmt(pt), dst_ty.fmt(pt), - }); - - var sym_buf: ["__fixuns?f?i".len]u8 = undefined; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = dst_ty.toIntern(), - .param_types = &.{src_ty.toIntern()}, - .sym = std.fmt.bufPrint(&sym_buf, "__fix{s}{c}f{c}i", .{ - switch (dst_signedness) { - .signed => "", - .unsigned => "uns", - }, - floatCompilerRtAbiName(src_bits), - intCompilerRtAbiName(dst_bits), - }) catch unreachable, - } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); - } - - const src_mcv = try self.resolveInst(ty_op.operand); - const src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(src_ty, src_mcv); - const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); - defer self.register_manager.unlockReg(src_lock); - - const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty)); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - try self.asmRegisterRegister( - switch (src_bits) { - 32 => if (self.hasFeature(.avx)) .{ .v_, .cvttss2si } else .{ ._, .cvttss2si }, - 64 => if (self.hasFeature(.avx)) .{ .v_, .cvttsd2si } else .{ ._, .cvttsd2si }, - else => unreachable, - }, - registerAlias(dst_reg, dst_size), - src_reg.to128(), - ); - - if (dst_bits < dst_size * 8) try self.truncateRegister(dst_ty, dst_reg); - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; - - const dst_ty = self.typeOfIndex(inst); - const ptr_ty = self.typeOf(extra.ptr); - const val_ty = self.typeOf(extra.expected_value); - const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu)); - - try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx }); - const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx }); - defer for (regs_lock) |lock| self.register_manager.unlockReg(lock); - - const exp_mcv = try self.resolveInst(extra.expected_value); - if (val_abi_size > 8) { - const exp_addr_mcv: MCValue = switch (exp_mcv) { - .memory, .indirect, .load_frame => exp_mcv.address(), - else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) }, - }; - const exp_addr_lock = - if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{}); - try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{}); - } else try self.genSetReg(.rax, val_ty, exp_mcv, .{}); - - const new_mcv = try self.resolveInst(extra.new_value); - const new_reg = if (val_abi_size > 8) new: { - const new_addr_mcv: MCValue = switch (new_mcv) { - .memory, .indirect, .load_frame => new_mcv.address(), - else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) }, - }; - const new_addr_lock = - if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; - defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{}); - try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{}); - break :new null; - } else try self.copyToTmpRegister(val_ty, new_mcv); - const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; - defer if (new_lock) |lock| self.register_manager.unlockReg(lock); - - const ptr_mcv = try self.resolveInst(extra.ptr); - const mem_size: Memory.Size = .fromSize(val_abi_size); - const ptr_mem: Memory = switch (ptr_mcv) { - .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }), - else => .{ - .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, - .mod = .{ .rm = .{ .size = mem_size } }, - }, - }; - switch (ptr_mem.mod) { - .rm => {}, - .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}), - } - const ptr_lock = switch (ptr_mem.base) { - .none, .frame, .nav, .uav => null, - .reg => |reg| self.register_manager.lockReg(reg), - .table, .rip_inst, .lazy_sym, .extern_func => unreachable, - }; - defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); - - try self.spillEflagsIfOccupied(); - const null_reg = if (dst_ty.optionalReprIsPayload(zcu) and !self.liveness.isUnused(inst)) null_reg: { - const null_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - try self.asmRegisterRegister(.{ ._, .xor }, null_reg.to32(), null_reg.to32()); - break :null_reg null_reg; - } else null; - const null_lock = if (null_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; - defer if (null_lock) |lock| self.register_manager.unlockReg(lock); - - if (val_abi_size <= 8) try self.asmMemoryRegister( - .{ .@"lock _", .cmpxchg }, - ptr_mem, - registerAlias(new_reg.?, val_abi_size), - ) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); - - const result: MCValue = result: { - if (self.liveness.isUnused(inst)) break :result .unreach; - - if (null_reg) |reg| try self.asmCmovccRegisterRegister( - .e, - registerAlias(.rax, val_abi_size), - registerAlias(reg, val_abi_size), - ); - - if (val_abi_size <= 8) { - if (null_reg) |_| break :result .{ .register = .rax }; - self.eflags_inst = inst; - break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } }; - } - - const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{}); - try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{}); - if (null_reg == null) try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{}); - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); -} - -fn atomicOp( - self: *CodeGen, - ptr_mcv: MCValue, - val_mcv: MCValue, - ptr_ty: Type, - val_ty: Type, - unused: bool, - rmw_op: ?std.builtin.AtomicRmwOp, - order: std.builtin.AtomicOrder, -) InnerError!MCValue { - const pt = self.pt; - const zcu = pt.zcu; - const ptr_lock = switch (ptr_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); - - const val_lock = switch (val_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (val_lock) |lock| self.register_manager.unlockReg(lock); - - const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu)); - const mem_size: Memory.Size = .fromSize(val_abi_size); - const ptr_mem: Memory = switch (ptr_mcv) { - .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }), - else => .{ - .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, - .mod = .{ .rm = .{ .size = mem_size } }, - }, - }; - switch (ptr_mem.mod) { - .rm => {}, - .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}), - } - const mem_lock = switch (ptr_mem.base) { - .none, .frame, .nav, .uav => null, - .reg => |reg| self.register_manager.lockReg(reg), - .table, .rip_inst, .lazy_sym, .extern_func => unreachable, - }; - defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); - - const use_sse = rmw_op orelse .Xchg != .Xchg and val_ty.isRuntimeFloat(); - const strat: enum { lock, loop, libcall } = if (use_sse) .loop else switch (rmw_op orelse .Xchg) { - .Xchg, - .Add, - .Sub, - => if (val_abi_size <= 8) .lock else if (val_abi_size <= 16) .loop else .libcall, - .And, - .Or, - .Xor, - => if (val_abi_size <= 8 and unused) .lock else if (val_abi_size <= 16) .loop else .libcall, - .Nand, - .Max, - .Min, - => if (val_abi_size <= 16) .loop else .libcall, - }; - switch (strat) { - .lock => { - const mir_tag: Mir.Inst.FixedTag = if (rmw_op) |op| switch (op) { - .Xchg => if (unused) .{ ._, .mov } else .{ ._g, .xch }, - .Add => .{ .@"lock _", if (unused) .add else .xadd }, - .Sub => .{ .@"lock _", if (unused) .sub else .xadd }, - .And => .{ .@"lock _", .@"and" }, - .Or => .{ .@"lock _", .@"or" }, - .Xor => .{ .@"lock _", .xor }, - else => unreachable, - } else switch (order) { - .unordered, .monotonic, .release, .acq_rel => .{ ._, .mov }, - .acquire => unreachable, - .seq_cst => .{ ._g, .xch }, - }; - - const dst_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const dst_mcv = MCValue{ .register = dst_reg }; - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - try self.genSetReg(dst_reg, val_ty, val_mcv, .{}); - if (rmw_op == std.builtin.AtomicRmwOp.Sub and mir_tag[1] == .xadd) { - try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv); - } - try self.asmMemoryRegister(mir_tag, ptr_mem, registerAlias(dst_reg, val_abi_size)); - - return if (unused) .unreach else dst_mcv; - }, - .loop => _ = if (val_abi_size <= 8) { - const sse_reg: Register = if (use_sse) - try self.register_manager.allocReg(null, abi.RegisterClass.sse) - else - undefined; - const sse_lock = - if (use_sse) self.register_manager.lockRegAssumeUnused(sse_reg) else undefined; - defer if (use_sse) self.register_manager.unlockReg(sse_lock); - - const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const tmp_mcv = MCValue{ .register = tmp_reg }; - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem); - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - if (!use_sse and rmw_op orelse .Xchg != .Xchg) { - try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax }, .{}); - } - if (rmw_op) |op| if (use_sse) { - const mir_tag = @as(?Mir.Inst.FixedTag, switch (op) { - .Add => switch (val_ty.floatBits(self.target)) { - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, - else => null, - }, - .Sub => switch (val_ty.floatBits(self.target)) { - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, - else => null, - }, - .Min => switch (val_ty.floatBits(self.target)) { - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, - else => null, - }, - .Max => switch (val_ty.floatBits(self.target)) { - 32 => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, - 64 => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, - else => null, - }, - else => unreachable, - }) orelse return self.fail("TODO implement atomicOp of {s} for {f}", .{ - @tagName(op), val_ty.fmt(pt), - }); - try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{}); - switch (mir_tag[0]) { - .v_ss, .v_sd => if (val_mcv.isBase()) try self.asmRegisterRegisterMemory( - mir_tag, - sse_reg.to128(), - sse_reg.to128(), - try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), - ) else try self.asmRegisterRegisterRegister( - mir_tag, - sse_reg.to128(), - sse_reg.to128(), - (if (val_mcv.isRegister()) - val_mcv.getReg().? - else - try self.copyToTmpRegister(val_ty, val_mcv)).to128(), - ), - ._ss, ._sd => if (val_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - sse_reg.to128(), - try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), - ) else try self.asmRegisterRegister( - mir_tag, - sse_reg.to128(), - (if (val_mcv.isRegister()) - val_mcv.getReg().? - else - try self.copyToTmpRegister(val_ty, val_mcv)).to128(), - ), - else => unreachable, - } - try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg }, .{}); - } else switch (op) { - .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv, .{}), - .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv), - .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv), - .And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv), - .Nand => { - try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv); - try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv); - }, - .Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv), - .Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv), - .Min, .Max => { - const cc: Condition = switch (if (val_ty.isAbiInt(zcu)) - val_ty.intInfo(zcu).signedness - else - .unsigned) { - .unsigned => switch (op) { - .Min => .a, - .Max => .b, - else => unreachable, - }, - .signed => switch (op) { - .Min => .g, - .Max => .l, - else => unreachable, - }, - }; - - const cmov_abi_size = @max(val_abi_size, 2); - switch (val_mcv) { - .register => |val_reg| { - try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv); - try self.asmCmovccRegisterRegister( - cc, - registerAlias(tmp_reg, cmov_abi_size), - registerAlias(val_reg, cmov_abi_size), - ); - }, - .memory, .indirect, .load_frame => { - try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv); - try self.asmCmovccRegisterMemory( - cc, - registerAlias(tmp_reg, cmov_abi_size), - try val_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }), - ); - }, - else => { - const mat_reg = try self.copyToTmpRegister(val_ty, val_mcv); - const mat_lock = self.register_manager.lockRegAssumeUnused(mat_reg); - defer self.register_manager.unlockReg(mat_lock); - - try self.genBinOpMir( - .{ ._, .cmp }, - val_ty, - tmp_mcv, - .{ .register = mat_reg }, - ); - try self.asmCmovccRegisterRegister( - cc, - registerAlias(tmp_reg, cmov_abi_size), - registerAlias(mat_reg, cmov_abi_size), - ); - }, - } - }, - }; - try self.asmMemoryRegister( - .{ .@"lock _", .cmpxchg }, - ptr_mem, - registerAlias(tmp_reg, val_abi_size), - ); - _ = try self.asmJccReloc(.ne, loop); - return if (unused) .unreach else .{ .register = .rax }; - } else { - try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{ - .base = ptr_mem.base, - .mod = .{ .rm = .{ - .size = .qword, - .index = ptr_mem.mod.rm.index, - .scale = ptr_mem.mod.rm.scale, - .disp = ptr_mem.mod.rm.disp + 0, - } }, - }); - try self.asmRegisterMemory(.{ ._, .mov }, .rdx, .{ - .base = ptr_mem.base, - .mod = .{ .rm = .{ - .size = .qword, - .index = ptr_mem.mod.rm.index, - .scale = ptr_mem.mod.rm.scale, - .disp = ptr_mem.mod.rm.disp + 8, - } }, - }); - const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - const val_mem_mcv: MCValue = switch (val_mcv) { - .memory, .indirect, .load_frame => val_mcv, - else => .{ .indirect = .{ - .reg = try self.copyToTmpRegister(.usize, val_mcv.address()), - } }, - }; - const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword }); - const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }); - if (rmw_op != std.builtin.AtomicRmwOp.Xchg) { - try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax); - try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx); - } - if (rmw_op) |op| switch (op) { - .Xchg => { - try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem); - }, - .Add => { - try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem); - }, - .Sub => { - try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem); - }, - .And => { - try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem); - }, - .Nand => { - try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem); - try self.asmRegister(.{ ._, .not }, .rbx); - try self.asmRegister(.{ ._, .not }, .rcx); - }, - .Or => { - try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem); - }, - .Xor => { - try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem); - }, - .Min, .Max => { - const cc: Condition = switch (if (val_ty.isAbiInt(zcu)) - val_ty.intInfo(zcu).signedness - else - .unsigned) { - .unsigned => switch (op) { - .Min => .a, - .Max => .b, - else => unreachable, - }, - .signed => switch (op) { - .Min => .g, - .Max => .l, - else => unreachable, - }, - }; - - const tmp_reg = try self.copyToTmpRegister(.usize, .{ .register = .rcx }); - const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); - defer self.register_manager.unlockReg(tmp_lock); - - try self.asmRegisterMemory(.{ ._, .cmp }, .rbx, val_lo_mem); - try self.asmRegisterMemory(.{ ._, .sbb }, tmp_reg, val_hi_mem); - try self.asmCmovccRegisterMemory(cc, .rbx, val_lo_mem); - try self.asmCmovccRegisterMemory(cc, .rcx, val_hi_mem); - }, - }; - try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); - _ = try self.asmJccReloc(.ne, loop); - - if (unused) return .unreach; - const dst_mcv = try self.allocTempRegOrMem(val_ty, false); - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_mcv.load_frame.off + 0, - } }, - }, .rax); - try self.asmMemoryRegister(.{ ._, .mov }, .{ - .base = .{ .frame = dst_mcv.load_frame.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = dst_mcv.load_frame.off + 8, - } }, - }, .rdx); - return dst_mcv; - }, - .libcall => return self.fail("TODO implement x86 atomic libcall", .{}), - } -} - -fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; - - try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx }); - const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx }); - defer for (regs_lock) |lock| self.register_manager.unlockReg(lock); - - const unused = self.liveness.isUnused(inst); - - const ptr_ty = self.typeOf(pl_op.operand); - const ptr_mcv = try self.resolveInst(pl_op.operand); - - const val_ty = self.typeOf(extra.operand); - const val_mcv = try self.resolveInst(extra.operand); - - const result = - try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering()); - return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); -} - -fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void { - const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; - const result: MCValue = result: { - const ptr_ty = self.typeOf(atomic_load.ptr); - const ptr_mcv = try self.resolveInst(atomic_load.ptr); - const ptr_lock = switch (ptr_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); - - const unused = self.liveness.isUnused(inst); - - const dst_mcv: MCValue = if (unused) - .{ .register = try self.register_manager.allocReg(null, self.regSetForType(ptr_ty.childType(self.pt.zcu))) } - else if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv)) - ptr_mcv - else - try self.allocRegOrMem(inst, true); - - try self.load(dst_mcv, ptr_ty, ptr_mcv); - - break :result if (unused) .unreach else dst_mcv; - }; - return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none }); -} - -fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const ptr_ty = self.typeOf(bin_op.lhs); - const ptr_mcv = try self.resolveInst(bin_op.lhs); - - const val_ty = self.typeOf(bin_op.rhs); - const val_mcv = try self.resolveInst(bin_op.rhs); - - const result = try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { - const pt = self.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - result: { - if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; - - try self.spillRegisters(&.{ .rax, .rdi, .rsi, .rcx }); - const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx }); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - const dst = try self.resolveInst(bin_op.lhs); - const dst_ty = self.typeOf(bin_op.lhs); - const dst_locks: [2]?RegisterLock = switch (dst) { - .register => |dst_reg| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null }, - .register_pair => |dst_regs| .{ - self.register_manager.lockRegAssumeUnused(dst_regs[0]), - self.register_manager.lockRegAssumeUnused(dst_regs[1]), - }, - else => @splat(null), - }; - for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const src_val = try self.resolveInst(bin_op.rhs); - const elem_ty = self.typeOf(bin_op.rhs); - const src_val_lock: ?RegisterLock = switch (src_val) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock); - - const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu)); - - if (elem_abi_size == 1) { - const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) { - .slice => switch (dst) { - .register_pair => |dst_regs| .{ .register = dst_regs[0] }, - else => dst, - }, - .one => dst, - .c, .many => unreachable, - }; - const len: MCValue = switch (dst_ty.ptrSize(zcu)) { - .slice => switch (dst) { - .register_pair => |dst_regs| .{ .register = dst_regs[1] }, - else => dst.address().offset(8).deref(), - }, - .one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) }, - .c, .many => unreachable, - }; - const len_lock: ?RegisterLock = switch (len) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (len_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety }); - break :result; - } - - // Store the first element, and then rely on memcpy copying forwards. - // Length zero requires a runtime check - so we handle arrays specially - // here to elide it. - switch (dst_ty.ptrSize(zcu)) { - .slice => { - const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu); - - const dst_ptr: MCValue = switch (dst) { - .register_pair => |dst_regs| .{ .register = dst_regs[0] }, - else => dst, - }; - const len: MCValue = switch (dst) { - .register_pair => |dst_regs| .{ .register = dst_regs[1] }, - else => dst.address().offset(8).deref(), - }; - - // Used to store the number of elements for comparison. - // After comparison, updated to store number of bytes needed to copy. - const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const len_mcv: MCValue = .{ .register = len_reg }; - const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); - defer self.register_manager.unlockReg(len_lock); - - try self.genSetReg(len_reg, .usize, len, .{}); - try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); - - const skip_reloc = try self.asmJccReloc(.z, undefined); - try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety }); - - const second_elem_ptr_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; - const second_elem_ptr_lock = - self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); - defer self.register_manager.unlockReg(second_elem_ptr_lock); - - try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{ - .reg = try self.copyToTmpRegister(.usize, dst_ptr), - .off = elem_abi_size, - } }, .{}); - - try self.genBinOpMir(.{ ._, .sub }, .usize, len_mcv, .{ .immediate = 1 }); - try self.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - len_reg, - len_reg, - .s(elem_abi_size), - ); - try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv, .{ .no_alias = false }); - - self.performReloc(skip_reloc); - }, - .one => { - const elem_ptr_ty = try pt.singleMutPtrType(elem_ty); - - const len = dst_ty.childType(zcu).arrayLen(zcu); - - assert(len != 0); // prevented by Sema - try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety }); - - const second_elem_ptr_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; - const second_elem_ptr_lock = - self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); - defer self.register_manager.unlockReg(second_elem_ptr_lock); - - try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{ - .reg = try self.copyToTmpRegister(.usize, dst), - .off = elem_abi_size, - } }, .{}); - - const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; - try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy, .{ .no_alias = false }); - }, - .c, .many => unreachable, - } - } - return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const vector_ty = self.typeOfIndex(inst); - const vector_len = vector_ty.vectorLen(zcu); - const dst_rc = self.regSetForType(vector_ty); - const scalar_ty = self.typeOf(ty_op.operand); - - const result: MCValue = result: { - switch (scalar_ty.zigTypeTag(zcu)) { - else => {}, - .bool => { - const regs = - try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs); - defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - - try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 }, .{}); - try self.genSetReg( - regs[1], - vector_ty, - .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) }, - .{}, - ); - const src_mcv = try self.resolveInst(ty_op.operand); - const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4); - try self.asmCmovccRegisterRegister( - switch (src_mcv) { - .eflags => |cc| cc, - .register => |src_reg| cc: { - try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1)); - break :cc .nz; - }, - else => cc: { - try self.asmMemoryImmediate( - .{ ._, .@"test" }, - try src_mcv.mem(self, .{ .size = .byte }), - .u(1), - ); - break :cc .nz; - }, - }, - registerAlias(regs[0], abi_size), - registerAlias(regs[1], abi_size), - ); - break :result .{ .register = regs[0] }; - }, - .int => if (self.hasFeature(.avx2)) avx2: { - const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(zcu).bits) { - else => null, - 1...8 => switch (vector_len) { - else => null, - 1...32 => .{ .vp_b, .broadcast }, - }, - 9...16 => switch (vector_len) { - else => null, - 1...16 => .{ .vp_w, .broadcast }, - }, - 17...32 => switch (vector_len) { - else => null, - 1...8 => .{ .vp_d, .broadcast }, - }, - 33...64 => switch (vector_len) { - else => null, - 1...4 => .{ .vp_q, .broadcast }, - }, - 65...128 => switch (vector_len) { - else => null, - 1...2 => .{ .v_i128, .broadcast }, - }, - }) orelse break :avx2; - - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - const src_mcv = try self.resolveInst(ty_op.operand); - if (src_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), - try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }), - ) else { - if (mir_tag[0] == .v_i128) break :avx2; - try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); - try self.asmRegisterRegister( - mir_tag, - registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), - registerAlias(dst_reg, @intCast(scalar_ty.abiSize(zcu))), - ); - } - break :result .{ .register = dst_reg }; - } else { - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - - try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand }, .{}); - if (vector_len == 1) break :result .{ .register = dst_reg }; - - const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))); - const scalar_bits = scalar_ty.intInfo(zcu).bits; - if (switch (scalar_bits) { - 1...8 => true, - 9...128 => false, - else => unreachable, - }) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister( - .{ .vp_, .unpcklbw }, - dst_alias, - dst_alias, - dst_alias, - ) else try self.asmRegisterRegister( - .{ .p_, .unpcklbw }, - dst_alias, - dst_alias, - ); - if (switch (scalar_bits) { - 1...8 => vector_len > 2, - 9...16 => true, - 17...128 => false, - else => unreachable, - }) try self.asmRegisterRegisterImmediate( - .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl }, - dst_alias, - dst_alias, - .u(0b00_00_00_00), - ); - if (switch (scalar_bits) { - 1...8 => vector_len > 4, - 9...16 => vector_len > 2, - 17...64 => true, - 65...128 => false, - else => unreachable, - }) try self.asmRegisterRegisterImmediate( - .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf }, - dst_alias, - dst_alias, - .u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00), - ); - break :result .{ .register = dst_reg }; - }, - .float => switch (scalar_ty.floatBits(self.target)) { - 32 => switch (vector_len) { - 1 => { - const src_mcv = try self.resolveInst(ty_op.operand); - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); - break :result .{ .register = dst_reg }; - }, - 2...4 => { - const src_mcv = try self.resolveInst(ty_op.operand); - if (self.hasFeature(.avx)) { - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_ss, .broadcast }, - dst_reg.to128(), - try src_mcv.mem(self, .{ .size = .dword }), - ) else { - const src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(scalar_ty, src_mcv); - try self.asmRegisterRegisterRegisterImmediate( - .{ .v_ps, .shuf }, - dst_reg.to128(), - src_reg.to128(), - src_reg.to128(), - .u(0), - ); - } - break :result .{ .register = dst_reg }; - } else { - const dst_mcv = if (src_mcv.isRegister() and - self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) - src_mcv - else - try self.copyToRegisterWithInstTracking(inst, scalar_ty, src_mcv); - const dst_reg = dst_mcv.getReg().?; - try self.asmRegisterRegisterImmediate( - .{ ._ps, .shuf }, - dst_reg.to128(), - dst_reg.to128(), - .u(0), - ); - break :result dst_mcv; - } - }, - 5...8 => if (self.hasFeature(.avx)) { - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_ss, .broadcast }, - dst_reg.to256(), - try src_mcv.mem(self, .{ .size = .dword }), - ) else { - const src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(scalar_ty, src_mcv); - if (self.hasFeature(.avx2)) try self.asmRegisterRegister( - .{ .v_ss, .broadcast }, - dst_reg.to256(), - src_reg.to128(), - ) else { - try self.asmRegisterRegisterRegisterImmediate( - .{ .v_ps, .shuf }, - dst_reg.to128(), - src_reg.to128(), - src_reg.to128(), - .u(0), - ); - try self.asmRegisterRegisterRegisterImmediate( - .{ .v_f128, .insert }, - dst_reg.to256(), - dst_reg.to256(), - dst_reg.to128(), - .u(1), - ); - } - } - break :result .{ .register = dst_reg }; - }, - else => {}, - }, - 64 => switch (vector_len) { - 1 => { - const src_mcv = try self.resolveInst(ty_op.operand); - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); - break :result .{ .register = dst_reg }; - }, - 2 => { - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - if (self.hasFeature(.sse3)) { - if (src_mcv.isBase()) try self.asmRegisterMemory( - if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, - dst_reg.to128(), - try src_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, - dst_reg.to128(), - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(), - ); - break :result .{ .register = dst_reg }; - } else try self.asmRegisterRegister( - .{ ._ps, .movlh }, - dst_reg.to128(), - (if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(), - ); - }, - 3...4 => if (self.hasFeature(.avx)) { - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_sd, .broadcast }, - dst_reg.to256(), - try src_mcv.mem(self, .{ .size = .qword }), - ) else { - const src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(scalar_ty, src_mcv); - if (self.hasFeature(.avx2)) try self.asmRegisterRegister( - .{ .v_sd, .broadcast }, - dst_reg.to256(), - src_reg.to128(), - ) else { - try self.asmRegisterRegister( - .{ .v_, .movddup }, - dst_reg.to128(), - src_reg.to128(), - ); - try self.asmRegisterRegisterRegisterImmediate( - .{ .v_f128, .insert }, - dst_reg.to256(), - dst_reg.to256(), - dst_reg.to128(), - .u(1), - ); - } - } - break :result .{ .register = dst_reg }; - }, - else => {}, - }, - 128 => switch (vector_len) { - 1 => { - const src_mcv = try self.resolveInst(ty_op.operand); - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); - break :result .{ .register = dst_reg }; - }, - 2 => if (self.hasFeature(.avx)) { - const src_mcv = try self.resolveInst(ty_op.operand); - const dst_reg = try self.register_manager.allocReg(inst, dst_rc); - if (src_mcv.isBase()) try self.asmRegisterMemory( - .{ .v_f128, .broadcast }, - dst_reg.to256(), - try src_mcv.mem(self, .{ .size = .xword }), - ) else { - const src_reg = if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(scalar_ty, src_mcv); - try self.asmRegisterRegisterRegisterImmediate( - .{ .v_f128, .insert }, - dst_reg.to256(), - src_reg.to256(), - src_reg.to128(), - .u(1), - ); - } - break :result .{ .register = dst_reg }; - }, - else => {}, - }, - 16, 80 => {}, - else => unreachable, - }, - } - return self.fail("TODO implement airSplat for {f}", .{vector_ty.fmt(pt)}); - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload).data; - const ty = self.typeOfIndex(inst); - const vec_len = ty.vectorLen(zcu); - const elem_ty = ty.childType(zcu); - const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const pred_ty = self.typeOf(pl_op.operand); - - const result = result: { - const has_blend = self.hasFeature(.sse4_1); - const has_avx = self.hasFeature(.avx); - const need_xmm0 = has_blend and !has_avx; - const pred_mcv = try self.resolveInst(pl_op.operand); - const mask_reg = mask: { - switch (pred_mcv) { - .register => |pred_reg| switch (pred_reg.class()) { - .general_purpose => {}, - .sse => if (elem_ty.toIntern() == .bool_type) - if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) { - try self.register_manager.getKnownReg(.xmm0, null); - try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{}); - break :mask .xmm0; - } else break :mask if (has_blend) - pred_reg - else - try self.copyToTmpRegister(pred_ty, pred_mcv) - else - return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}), - else => unreachable, - }, - .register_mask => |pred_reg_mask| { - if (pred_reg_mask.info.scalar.bitSize(self.target) != 8 * elem_abi_size) - return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); - - const mask_reg: Register = if (need_xmm0 and pred_reg_mask.reg.id() != comptime Register.xmm0.id()) mask_reg: { - try self.register_manager.getKnownReg(.xmm0, null); - try self.genSetReg(.xmm0, ty, .{ .register = pred_reg_mask.reg }, .{}); - break :mask_reg .xmm0; - } else pred_reg_mask.reg; - const mask_alias = registerAlias(mask_reg, abi_size); - const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); - defer self.register_manager.unlockReg(mask_lock); - - const lhs_mcv = try self.resolveInst(extra.lhs); - const lhs_lock = switch (lhs_mcv) { - .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(extra.rhs); - const rhs_lock = switch (rhs_mcv) { - .register => |rhs_reg| self.register_manager.lockReg(rhs_reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const order = has_blend != pred_reg_mask.info.inverted; - const reuse_mcv, const other_mcv = if (order) - .{ rhs_mcv, lhs_mcv } - else - .{ lhs_mcv, rhs_mcv }; - const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand( - inst, - if (order) extra.rhs else extra.lhs, - @intFromBool(order), - reuse_mcv, - )) reuse_mcv else if (has_avx) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, abi_size); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const mir_tag = @as(?Mir.Inst.FixedTag, if ((pred_reg_mask.info.kind == .all and - elem_ty.toIntern() != .f32_type and elem_ty.toIntern() != .f64_type) or pred_reg_mask.info.scalar == .byte) - if (has_avx) - .{ .vp_b, .blendv } - else if (has_blend) - .{ .p_b, .blendv } - else if (pred_reg_mask.info.kind == .all) - .{ .p_, undefined } - else - null - else if ((pred_reg_mask.info.kind == .all and (elem_ty.toIntern() != .f64_type or !self.hasFeature(.sse2))) or - pred_reg_mask.info.scalar == .dword) - if (has_avx) - .{ .v_ps, .blendv } - else if (has_blend) - .{ ._ps, .blendv } - else if (pred_reg_mask.info.kind == .all) - .{ ._ps, undefined } - else - null - else if (pred_reg_mask.info.kind == .all or pred_reg_mask.info.scalar == .qword) - if (has_avx) - .{ .v_pd, .blendv } - else if (has_blend) - .{ ._pd, .blendv } - else if (pred_reg_mask.info.kind == .all) - .{ ._pd, undefined } - else - null - else - null) orelse return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); - if (has_avx) { - const rhs_alias = if (reuse_mcv.isRegister()) - registerAlias(reuse_mcv.getReg().?, abi_size) - else rhs: { - try self.genSetReg(dst_reg, ty, reuse_mcv, .{}); - break :rhs dst_alias; - }; - if (other_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( - mir_tag, - dst_alias, - rhs_alias, - try other_mcv.mem(self, .{ .size = self.memSize(ty) }), - mask_alias, - ) else try self.asmRegisterRegisterRegisterRegister( - mir_tag, - dst_alias, - rhs_alias, - registerAlias(if (other_mcv.isRegister()) - other_mcv.getReg().? - else - try self.copyToTmpRegister(ty, other_mcv), abi_size), - mask_alias, - ); - } else if (has_blend) if (other_mcv.isBase()) try self.asmRegisterMemoryRegister( - mir_tag, - dst_alias, - try other_mcv.mem(self, .{ .size = self.memSize(ty) }), - mask_alias, - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (other_mcv.isRegister()) - other_mcv.getReg().? - else - try self.copyToTmpRegister(ty, other_mcv), abi_size), - mask_alias, - ) else { - try self.asmRegisterRegister(.{ mir_tag[0], .@"and" }, dst_alias, mask_alias); - if (other_mcv.isBase()) try self.asmRegisterMemory( - .{ mir_tag[0], .andn }, - mask_alias, - try other_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - ) else try self.asmRegisterRegister( - .{ mir_tag[0], .andn }, - mask_alias, - registerAlias(if (other_mcv.isRegister()) - other_mcv.getReg().? - else - try self.copyToTmpRegister(ty, other_mcv), abi_size), - ); - try self.asmRegisterRegister(.{ mir_tag[0], .@"or" }, dst_alias, mask_alias); - } - break :result dst_mcv; - }, - else => {}, - } - const mask_reg: Register = if (need_xmm0) mask_reg: { - try self.register_manager.getKnownReg(.xmm0, null); - break :mask_reg .xmm0; - } else try self.register_manager.allocReg(null, abi.RegisterClass.sse); - const mask_alias = registerAlias(mask_reg, abi_size); - const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); - defer self.register_manager.unlockReg(mask_lock); - - const pred_fits_in_elem = vec_len <= elem_abi_size; - if (self.hasFeature(.avx2) and abi_size <= 32) { - if (pred_mcv.isRegister()) broadcast: { - try self.asmRegisterRegister( - .{ .v_d, .mov }, - mask_reg.to128(), - pred_mcv.getReg().?.to32(), - ); - if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister( - .{ switch (elem_abi_size) { - 1 => .vp_b, - 2 => .vp_w, - 3...4 => .vp_d, - 5...8 => .vp_q, - 9...16 => { - try self.asmRegisterRegisterRegisterImmediate( - .{ .v_f128, .insert }, - mask_alias, - mask_alias, - mask_reg.to128(), - .u(1), - ); - break :broadcast; - }, - 17...32 => break :broadcast, - else => unreachable, - }, .broadcast }, - mask_alias, - mask_reg.to128(), - ); - } else try self.asmRegisterMemory( - .{ switch (vec_len) { - 1...8 => .vp_b, - 9...16 => .vp_w, - 17...32 => .vp_d, - else => unreachable, - }, .broadcast }, - mask_alias, - if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{ - .base = .{ .reg = (try self.copyToTmpRegister( - .usize, - pred_mcv.address(), - )).to64() }, - .mod = .{ .rm = .{ .size = .byte } }, - }, - ); - } else if (abi_size <= 16) broadcast: { - try self.asmRegisterRegister( - .{ if (has_avx) .v_d else ._d, .mov }, - mask_alias, - (if (pred_mcv.isRegister()) - pred_mcv.getReg().? - else - try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(), - ); - if (!pred_fits_in_elem or vec_len == 1) break :broadcast; - if (elem_abi_size <= 1) { - if (has_avx) try self.asmRegisterRegisterRegister( - .{ .vp_, .unpcklbw }, - mask_alias, - mask_alias, - mask_alias, - ) else try self.asmRegisterRegister( - .{ .p_, .unpcklbw }, - mask_alias, - mask_alias, - ); - if (abi_size <= 2) break :broadcast; - } - if (elem_abi_size <= 2) { - try self.asmRegisterRegisterImmediate( - .{ if (has_avx) .vp_w else .p_w, .shufl }, - mask_alias, - mask_alias, - .u(0b00_00_00_00), - ); - if (abi_size <= 8) break :broadcast; - } - try self.asmRegisterRegisterImmediate( - .{ if (has_avx) .vp_d else .p_d, .shuf }, - mask_alias, - mask_alias, - .u(switch (elem_abi_size) { - 1...2, 5...8 => 0b01_00_01_00, - 3...4 => 0b00_00_00_00, - else => unreachable, - }), - ); - } else return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); - const elem_bits: u16 = @intCast(elem_abi_size * 8); - if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) { - const mask_len = elem_abi_size * vec_len; - const mask_ty = try pt.vectorType(.{ - .len = mask_len, - .child = .u8_type, - }); - var mask_elems_buf: [32]u8 = undefined; - const mask_elems = mask_elems_buf[0..mask_len]; - for (mask_elems, 0..) |*elem, bit| elem.* = @intCast(bit / elem_bits); - const mask_mcv = try self.lowerValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = mask_ty.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, mask_elems, .maybe_embedded_nulls) }, - } }))); - const mask_mem: Memory = .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }; - if (has_avx) try self.asmRegisterRegisterMemory( - .{ .vp_b, .shuf }, - mask_alias, - mask_alias, - mask_mem, - ) else try self.asmRegisterMemory( - .{ .p_b, .shuf }, - mask_alias, - mask_mem, - ); - } else return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); - { - const mask_elem_ty = try pt.intType(.unsigned, elem_bits); - const mask_ty = try pt.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() }); - var mask_elems_buf: [32]InternPool.Index = undefined; - const mask_elems = mask_elems_buf[0..vec_len]; - for (mask_elems, 0..) |*elem, bit| elem.* = (try pt.intValue( - mask_elem_ty, - @as(u8, 1) << @truncate(bit), - )).toIntern(); - const mask_mcv = try self.lowerValue(try pt.aggregateValue(mask_ty, mask_elems)); - const mask_mem: Memory = .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }; - if (has_avx) { - try self.asmRegisterRegisterMemory( - .{ .vp_, .@"and" }, - mask_alias, - mask_alias, - mask_mem, - ); - try self.asmRegisterRegisterMemory( - .{ switch (elem_abi_size) { - else => unreachable, - 1 => .vp_b, - 2 => .vp_w, - 4 => .vp_d, - 8 => .vp_q, - }, .cmpeq }, - mask_alias, - mask_alias, - mask_mem, - ); - } else { - try self.asmRegisterMemory( - .{ .p_, .@"and" }, - mask_alias, - mask_mem, - ); - try self.asmRegisterMemory( - .{ switch (elem_abi_size) { - else => unreachable, - 1 => .p_b, - 2 => .p_w, - 4 => .p_d, - 8 => if (has_blend) .p_q else unreachable, - }, .cmpeq }, - mask_alias, - mask_mem, - ); - } - } - break :mask mask_reg; - }; - const mask_alias = registerAlias(mask_reg, abi_size); - const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); - defer self.register_manager.unlockReg(mask_lock); - - const lhs_mcv = try self.resolveInst(extra.lhs); - const lhs_lock = switch (lhs_mcv) { - .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(extra.rhs); - const rhs_lock = switch (rhs_mcv) { - .register => |rhs_reg| self.register_manager.lockReg(rhs_reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv; - const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand( - inst, - if (has_blend) extra.rhs else extra.lhs, - @intFromBool(has_blend), - reuse_mcv, - )) reuse_mcv else if (has_avx) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, abi_size); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - const mir_tag = @as(?Mir.Inst.FixedTag, switch (elem_ty.zigTypeTag(zcu)) { - else => null, - .int => switch (abi_size) { - 0 => unreachable, - 1...16 => if (has_avx) - .{ .vp_b, .blendv } - else if (has_blend) - .{ .p_b, .blendv } - else - .{ .p_, undefined }, - 17...32 => if (self.hasFeature(.avx2)) - .{ .vp_b, .blendv } - else - null, - else => null, - }, - .float => switch (elem_ty.floatBits(self.target)) { - else => unreachable, - 16, 80, 128 => null, - 32 => switch (vec_len) { - 0 => unreachable, - 1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv }, - 5...8 => if (has_avx) .{ .v_ps, .blendv } else null, - else => null, - }, - 64 => switch (vec_len) { - 0 => unreachable, - 1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv }, - 3...4 => if (has_avx) .{ .v_pd, .blendv } else null, - else => null, - }, - }, - }) orelse return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); - if (has_avx) { - const rhs_alias = if (rhs_mcv.isRegister()) - registerAlias(rhs_mcv.getReg().?, abi_size) - else rhs: { - try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); - break :rhs dst_alias; - }; - if (lhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( - mir_tag, - dst_alias, - rhs_alias, - try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), - mask_alias, - ) else try self.asmRegisterRegisterRegisterRegister( - mir_tag, - dst_alias, - rhs_alias, - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - try self.copyToTmpRegister(ty, lhs_mcv), abi_size), - mask_alias, - ); - } else if (has_blend) if (lhs_mcv.isBase()) try self.asmRegisterMemoryRegister( - mir_tag, - dst_alias, - try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), - mask_alias, - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - try self.copyToTmpRegister(ty, lhs_mcv), abi_size), - mask_alias, - ) else { - try self.asmRegisterRegister(.{ mir_tag[0], .@"and" }, dst_alias, mask_alias); - if (rhs_mcv.isBase()) try self.asmRegisterMemory( - .{ mir_tag[0], .andn }, - mask_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }), - ) else try self.asmRegisterRegister( - .{ mir_tag[0], .andn }, - mask_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(ty, rhs_mcv), abi_size), - ); - try self.asmRegisterRegister(.{ mir_tag[0], .@"or" }, dst_alias, mask_alias); - } - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); -} - -fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data; - - const dst_ty = self.typeOfIndex(inst); - const elem_ty = dst_ty.childType(zcu); - const elem_abi_size: u16 = @intCast(elem_ty.abiSize(zcu)); - const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const lhs_ty = self.typeOf(extra.a); - const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); - const rhs_ty = self.typeOf(extra.b); - const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu)); - const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size); - - const ExpectedContents = [32]?i32; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - const allocator = stack.get(); - - const mask_elems = try allocator.alloc(?i32, extra.mask_len); - defer allocator.free(mask_elems); - for (mask_elems, 0..) |*mask_elem, elem_index| { - const mask_elem_val = - Value.fromInterned(extra.mask).elemValue(pt, elem_index) catch unreachable; - mask_elem.* = if (mask_elem_val.isUndef(zcu)) - null - else - @intCast(mask_elem_val.toSignedInt(zcu)); - } - - const has_avx = self.hasFeature(.avx); - const result = @as(?MCValue, result: { - for (mask_elems) |mask_elem| { - if (mask_elem) |_| break; - } else break :result try self.allocRegOrMem(inst, true); - - for (mask_elems, 0..) |mask_elem, elem_index| { - if (mask_elem orelse continue != elem_index) break; - } else { - const lhs_mcv = try self.resolveInst(extra.a); - if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv; - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{}); - break :result dst_mcv; - } - - for (mask_elems, 0..) |mask_elem, elem_index| { - if (~(mask_elem orelse continue) != elem_index) break; - } else { - const rhs_mcv = try self.resolveInst(extra.b); - if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv; - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{}); - break :result dst_mcv; - } - - for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { - if (elem_abi_size > 8) break :unpck; - if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck; - - var sources: [2]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index = - std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; - const elem_byte = (elem_index >> 1) * elem_abi_size; - if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) { - .unpckl => 0b0000, - .unpckh => 0b1000, - else => unreachable, - }) | (elem_byte << 1 & 0b10000)) break :unpck; - - const source = @intFromBool(mask_elem < 0); - if (sources[elem_index & 0b00001]) |prev_source| { - if (source != prev_source) break :unpck; - } else sources[elem_index & 0b00001] = source; - } - if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck; - - const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; - const operand_tys = [2]Type{ lhs_ty, rhs_ty }; - const lhs_mcv = try self.resolveInst(operands[sources[0].?]); - const rhs_mcv = try self.resolveInst(operands[sources[1].?]); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, max_abi_size); - - const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or - (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) { - 4 => if (has_avx) .v_ps else ._ps, - 8 => if (has_avx) .v_pd else ._pd, - else => unreachable, - }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) { - .unpckl => switch (elem_abi_size) { - 1 => .unpcklbw, - 2 => .unpcklwd, - 4 => .unpckldq, - 8 => .unpcklqdq, - else => unreachable, - }, - .unpckh => switch (elem_abi_size) { - 1 => .unpckhbw, - 2 => .unpckhwd, - 4 => .unpckhdq, - 8 => .unpckhqdq, - else => unreachable, - }, - else => unreachable, - } }; - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory( - mir_tag, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - ); - break :result dst_mcv; - } - - pshufd: { - if (elem_abi_size != 4) break :pshufd; - if (max_abi_size > self.vectorSize(.float)) break :pshufd; - - var control: u8 = 0b00_00_00_00; - var sources: [1]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); - if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd; - - const source = @intFromBool(mask_elem < 0); - if (sources[0]) |prev_source| { - if (source != prev_source) break :pshufd; - } else sources[(elem_index & 0b010) >> 1] = source; - - const select_bit: u3 = @intCast((elem_index & 0b011) << 1); - const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; - if (elem_index & 0b100 == 0) - control |= select_mask - else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd; - } - - const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; - const operand_tys = [2]Type{ lhs_ty, rhs_ty }; - const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]); - - const dst_reg = if (src_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv)) - src_mcv.getReg().? - else - try self.register_manager.allocReg(inst, abi.RegisterClass.sse); - const dst_alias = registerAlias(dst_reg, max_abi_size); - - if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ if (has_avx) .vp_d else .p_d, .shuf }, - dst_alias, - try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterImmediate( - .{ if (has_avx) .vp_d else .p_d, .shuf }, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size), - .u(control), - ); - break :result .{ .register = dst_reg }; - } - - shufps: { - if (elem_abi_size != 4) break :shufps; - if (max_abi_size > self.vectorSize(.float)) break :shufps; - - var control: u8 = 0b00_00_00_00; - var sources: [2]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); - if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps; - - const source = @intFromBool(mask_elem < 0); - if (sources[(elem_index & 0b010) >> 1]) |prev_source| { - if (source != prev_source) break :shufps; - } else sources[(elem_index & 0b010) >> 1] = source; - - const select_bit: u3 = @intCast((elem_index & 0b011) << 1); - const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; - if (elem_index & 0b100 == 0) - control |= select_mask - else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps; - } - if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps; - - const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; - const operand_tys = [2]Type{ lhs_ty, rhs_ty }; - const lhs_mcv = try self.resolveInst(operands[sources[0].?]); - const rhs_mcv = try self.resolveInst(operands[sources[1].?]); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, max_abi_size); - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .v_ps, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .v_ps, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ ._ps, .shuf }, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterImmediate( - .{ ._ps, .shuf }, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ); - break :result dst_mcv; - } - - shufpd: { - if (elem_abi_size != 8) break :shufpd; - if (max_abi_size > self.vectorSize(.float)) break :shufpd; - - var control: u4 = 0b0_0_0_0; - var sources: [2]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); - if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd; - - const source = @intFromBool(mask_elem < 0); - if (sources[elem_index & 0b01]) |prev_source| { - if (source != prev_source) break :shufpd; - } else sources[elem_index & 0b01] = source; - - control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index); - } - if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd; - - const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b }; - const operand_tys: [2]Type = .{ lhs_ty, rhs_ty }; - const lhs_mcv = try self.resolveInst(operands[sources[0].?]); - const rhs_mcv = try self.resolveInst(operands[sources[1].?]); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, max_abi_size); - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .v_pd, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .v_pd, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ ._pd, .shuf }, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterImmediate( - .{ ._pd, .shuf }, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ); - break :result dst_mcv; - } - - blend: { - if (elem_abi_size < 2) break :blend; - if (dst_abi_size > self.vectorSize(.float)) break :blend; - if (!self.hasFeature(.sse4_1)) break :blend; - - var control: u8 = 0b0_0_0_0_0_0_0_0; - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index = - std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; - if (mask_elem_index != elem_index) break :blend; - - const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); - if (elem_index & 0b1000 == 0) - control |= select_mask - else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend; - } - - if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: { - const expanded_control = switch (elem_abi_size) { - 4 => control, - 8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | - @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00), - else => break :vpblendd, - }; - - const lhs_mcv = try self.resolveInst(extra.a); - const lhs_reg = if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, lhs_mcv); - const lhs_lock = self.register_manager.lockReg(lhs_reg); - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(extra.b); - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); - if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_d, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(lhs_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .vp_d, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(lhs_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ); - break :result .{ .register = dst_reg }; - } - - if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: { - const expanded_control = switch (elem_abi_size) { - 2 => control, - 4 => if (dst_abi_size <= 16 or - @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0))) - @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | - @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00) - else - break :pblendw, - 8 => if (dst_abi_size <= 16 or - @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0))) - @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) | - @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000) - else - break :pblendw, - 16 => break :pblendw, - else => unreachable, - }; - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, extra.a, 0, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .vp_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ .p_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterImmediate( - .{ .p_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ); - break :result .{ .register = dst_reg }; - } - - const expanded_control = switch (elem_abi_size) { - 4, 8 => control, - 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) | - @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00), - else => unreachable, - }; - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, extra.a, 0, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - switch (elem_abi_size) { - 4 => .{ .v_ps, .blend }, - 8, 16 => .{ .v_pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterRegisterImmediate( - switch (elem_abi_size) { - 4 => .{ .v_ps, .blend }, - 8, 16 => .{ .v_pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - switch (elem_abi_size) { - 4 => .{ ._ps, .blend }, - 8, 16 => .{ ._pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterImmediate( - switch (elem_abi_size) { - 4 => .{ ._ps, .blend }, - 8, 16 => .{ ._pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ); - break :result .{ .register = dst_reg }; - } - - blendv: { - if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv; - - const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); - const select_mask_ty = try pt.vectorType(.{ - .len = @intCast(mask_elems.len), - .child = select_mask_elem_ty.toIntern(), - }); - var select_mask_elems: [32]InternPool.Index = undefined; - for ( - select_mask_elems[0..mask_elems.len], - mask_elems, - 0.., - ) |*select_mask_elem, maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index = - std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; - if (mask_elem_index != elem_index) break :blendv; - - select_mask_elem.* = (if (mask_elem < 0) - try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty) - else - try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern(); - } - const select_mask_mcv = try self.lowerValue( - try pt.aggregateValue(select_mask_ty, select_mask_elems[0..mask_elems.len]), - ); - - if (self.hasFeature(.sse4_1)) { - const mir_tag: Mir.Inst.FixedTag = .{ - if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or - (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) { - 4 => if (has_avx) .v_ps else ._ps, - 8 => if (has_avx) .v_pd else ._pd, - else => unreachable, - } else if (has_avx) .vp_b else .p_b, - .blendv, - }; - - const select_mask_reg = if (!has_avx) reg: { - try self.register_manager.getKnownReg(.xmm0, null); - try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{}); - break :reg .xmm0; - } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); - const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size); - const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg); - defer self.register_manager.unlockReg(select_mask_lock); - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, extra.a, 0, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, dst_abi_size); - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( - mir_tag, - dst_alias, - if (lhs_mcv.isRegister()) - registerAlias(lhs_mcv.getReg().?, dst_abi_size) - else - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - select_mask_alias, - ) else try self.asmRegisterRegisterRegisterRegister( - mir_tag, - dst_alias, - if (lhs_mcv.isRegister()) - registerAlias(lhs_mcv.getReg().?, dst_abi_size) - else - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - select_mask_alias, - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister( - mir_tag, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - select_mask_alias, - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - select_mask_alias, - ); - break :result dst_mcv; - } - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (rhs_mcv.isRegister() and - self.reuseOperand(inst, extra.b, 1, rhs_mcv)) - rhs_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, dst_abi_size); - - const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); - const mask_alias = registerAlias(mask_reg, dst_abi_size); - const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); - defer self.register_manager.unlockReg(mask_lock); - - const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat()) - switch (elem_ty.floatBits(self.target)) { - 16, 80, 128 => .p_, - 32 => ._ps, - 64 => ._pd, - else => unreachable, - } - else - .p_; - try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); - if (lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ mir_fixes, .andn }, - mask_alias, - try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - ) else try self.asmRegisterRegister( - .{ mir_fixes, .andn }, - mask_alias, - if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, lhs_mcv), - ); - try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); - break :result dst_mcv; - } - - pshufb: { - if (max_abi_size > 16) break :pshufb; - if (!self.hasFeature(.ssse3)) break :pshufb; - - const temp_regs = - try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse); - const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs); - defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); - - const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size); - try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{}); - - const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size); - try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{}); - - var lhs_mask_elems: [16]InternPool.Index = undefined; - for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| { - const elem_index = byte_index / elem_abi_size; - lhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { - const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; - if (mask_elem < 0) break :elem 0b1_00_00000; - const mask_elem_index: u31 = @intCast(mask_elem); - const byte_off: u32 = @intCast(byte_index % elem_abi_size); - break :elem mask_elem_index * elem_abi_size + byte_off; - })).toIntern(); - } - const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); - const lhs_mask_mcv = try self.lowerValue( - try pt.aggregateValue(lhs_mask_ty, lhs_mask_elems[0..max_abi_size]), - ); - const lhs_mask_mem: Memory = .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) }, - .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, - }; - if (has_avx) try self.asmRegisterRegisterMemory( - .{ .vp_b, .shuf }, - lhs_temp_alias, - lhs_temp_alias, - lhs_mask_mem, - ) else try self.asmRegisterMemory( - .{ .p_b, .shuf }, - lhs_temp_alias, - lhs_mask_mem, - ); - - var rhs_mask_elems: [16]InternPool.Index = undefined; - for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| { - const elem_index = byte_index / elem_abi_size; - rhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { - const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; - if (mask_elem >= 0) break :elem 0b1_00_00000; - const mask_elem_index: u31 = @intCast(~mask_elem); - const byte_off: u32 = @intCast(byte_index % elem_abi_size); - break :elem mask_elem_index * elem_abi_size + byte_off; - })).toIntern(); - } - const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); - const rhs_mask_mcv = try self.lowerValue( - try pt.aggregateValue(rhs_mask_ty, rhs_mask_elems[0..max_abi_size]), - ); - const rhs_mask_mem: Memory = .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) }, - .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, - }; - if (has_avx) try self.asmRegisterRegisterMemory( - .{ .vp_b, .shuf }, - rhs_temp_alias, - rhs_temp_alias, - rhs_mask_mem, - ) else try self.asmRegisterMemory( - .{ .p_b, .shuf }, - rhs_temp_alias, - rhs_mask_mem, - ); - - if (has_avx) try self.asmRegisterRegisterRegister( - .{ switch (elem_ty.zigTypeTag(zcu)) { - else => break :result null, - .int => .vp_, - .float => switch (elem_ty.floatBits(self.target)) { - 32 => .v_ps, - 64 => .v_pd, - 16, 80, 128 => break :result null, - else => unreachable, - }, - }, .@"or" }, - lhs_temp_alias, - lhs_temp_alias, - rhs_temp_alias, - ) else try self.asmRegisterRegister( - .{ switch (elem_ty.zigTypeTag(zcu)) { - else => break :result null, - .int => .p_, - .float => switch (elem_ty.floatBits(self.target)) { - 32 => ._ps, - 64 => ._pd, - 16, 80, 128 => break :result null, - else => unreachable, - }, - }, .@"or" }, - lhs_temp_alias, - rhs_temp_alias, - ); - break :result .{ .register = temp_regs[0] }; - } - - break :result null; - }) orelse return self.fail("TODO implement airShuffle from {f} and {f} to {f} with {f}", .{ - lhs_ty.fmt(pt), - rhs_ty.fmt(pt), - dst_ty.fmt(pt), - Value.fromInterned(extra.mask).fmtValue(pt), - }); - return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); -} - -fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const result_ty = self.typeOfIndex(inst); - const len: usize = @intCast(result_ty.arrayLen(zcu)); - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); - const result: MCValue = result: { - switch (result_ty.zigTypeTag(zcu)) { - .@"struct" => { - const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu)); - if (result_ty.containerLayout(zcu) == .@"packed") { - const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern()); - try self.genInlineMemset( - .{ .lea_frame = .{ .index = frame_index } }, - .{ .immediate = 0 }, - .{ .immediate = result_ty.abiSize(zcu) }, - .{}, - ); - for (elements, 0..) |elem, elem_i_usize| { - const elem_i: u32 = @intCast(elem_i_usize); - if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; - - const elem_ty = result_ty.fieldType(elem_i, zcu); - const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu)); - if (elem_bit_size > 64) { - return self.fail( - "TODO airAggregateInit implement packed structs with large fields", - .{}, - ); - } - const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); - const elem_abi_bits = elem_abi_size * 8; - const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i); - const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size); - const elem_bit_off = elem_off % elem_abi_bits; - const elem_mcv = try self.resolveInst(elem); - const elem_lock = switch (elem_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - .immediate => |imm| lock: { - if (imm == 0) continue; - break :lock null; - }, - else => null, - }; - defer if (elem_lock) |lock| self.register_manager.unlockReg(lock); - - const elem_extra_bits = self.regExtraBits(elem_ty); - { - const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); - const temp_alias = registerAlias(temp_reg, elem_abi_size); - const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); - defer self.register_manager.unlockReg(temp_lock); - - if (elem_bit_off < elem_extra_bits) { - try self.truncateRegister(elem_ty, temp_alias); - } - if (elem_bit_off > 0) try self.genShiftBinOpMir( - .{ ._l, .sh }, - elem_ty, - .{ .register = temp_alias }, - .u8, - .{ .immediate = elem_bit_off }, - ); - try self.genBinOpMir( - .{ ._, .@"or" }, - elem_ty, - .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } }, - .{ .register = temp_alias }, - ); - } - if (elem_bit_off > elem_extra_bits) { - const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); - const temp_alias = registerAlias(temp_reg, elem_abi_size); - const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); - defer self.register_manager.unlockReg(temp_lock); - - if (elem_extra_bits > 0) { - try self.truncateRegister(elem_ty, temp_alias); - } - try self.genShiftBinOpMir( - .{ ._r, .sh }, - elem_ty, - .{ .register = temp_reg }, - .u8, - .{ .immediate = elem_abi_bits - elem_bit_off }, - ); - try self.genBinOpMir( - .{ ._, .@"or" }, - elem_ty, - .{ .load_frame = .{ - .index = frame_index, - .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)), - } }, - .{ .register = temp_alias }, - ); - } - } - } else for (elements, 0..) |elem, elem_i| { - if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; - - const elem_ty = result_ty.fieldType(elem_i, zcu); - const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu)); - const elem_mcv = try self.resolveInst(elem); - try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv, .{}); - } - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - .array, .vector => { - const elem_ty = result_ty.childType(zcu); - if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) { - const result_size: u32 = @intCast(result_ty.abiSize(zcu)); - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - try self.asmRegisterRegister( - .{ ._, .xor }, - registerAlias(dst_reg, @min(result_size, 4)), - registerAlias(dst_reg, @min(result_size, 4)), - ); - - for (elements, 0..) |elem, elem_i| { - const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem }); - const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); - defer self.register_manager.unlockReg(elem_lock); - - try self.asmRegisterImmediate( - .{ ._, .@"and" }, - registerAlias(elem_reg, @min(result_size, 4)), - .u(1), - ); - if (elem_i > 0) try self.asmRegisterImmediate( - .{ ._l, .sh }, - registerAlias(elem_reg, result_size), - .u(@intCast(elem_i)), - ); - try self.asmRegisterRegister( - .{ ._, .@"or" }, - registerAlias(dst_reg, result_size), - registerAlias(elem_reg, result_size), - ); - } - break :result .{ .register = dst_reg }; - } else { - const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu)); - const elem_size: u32 = @intCast(elem_ty.abiSize(zcu)); - - for (elements, 0..) |elem, elem_i| { - const elem_mcv = try self.resolveInst(elem); - const elem_off: i32 = @intCast(elem_size * elem_i); - try self.genSetMem( - .{ .frame = frame_index }, - elem_off, - elem_ty, - elem_mcv, - .{}, - ); - } - if (result_ty.sentinel(zcu)) |sentinel| try self.genSetMem( - .{ .frame = frame_index }, - @intCast(elem_size * elements.len), - elem_ty, - try self.lowerValue(sentinel), - .{}, - ); - break :result .{ .load_frame = .{ .index = frame_index } }; - } - }, - else => unreachable, - } - }; - - if (elements.len <= Air.Liveness.bpi - 1) { - var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); - @memcpy(buf[0..elements.len], elements); - return self.finishAir(inst, result, buf); - } - var bt = self.liveness.iterateBigTomb(inst); - for (elements) |elem| try self.feed(&bt, elem); - return self.finishAirResult(inst, result); -} - -fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; - const result: MCValue = result: { - const union_ty = self.typeOfIndex(inst); - const layout = union_ty.unionGetLayout(zcu); - - const src_ty = self.typeOf(extra.init); - const src_mcv = try self.resolveInst(extra.init); - if (layout.tag_size == 0) { - if (layout.abi_size <= src_ty.abiSize(zcu) and - self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv; - - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); - break :result dst_mcv; - } - - const dst_mcv = try self.allocRegOrMem(inst, false); - - const loaded_union = zcu.typeToUnion(union_ty).?; - const field_name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index]; - const tag_ty: Type = .fromInterned(loaded_union.enum_tag_ty); - const field_index = tag_ty.enumFieldIndex(field_name, zcu).?; - const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index); - const tag_int_val = try tag_val.intFromEnum(tag_ty, pt); - const tag_int = tag_int_val.toUnsignedInt(zcu); - const tag_off: i32 = @intCast(layout.tagOffset()); - try self.genCopy( - tag_ty, - dst_mcv.address().offset(tag_off).deref(), - .{ .immediate = tag_int }, - .{}, - ); - - const pl_off: i32 = @intCast(layout.payloadOffset()); - try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{}); - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ extra.init, .none, .none }); -} - -fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload).data; - const ty = self.typeOfIndex(inst); - - const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand }; - const result = result: { - if (switch (ty.scalarType(zcu).floatBits(self.target)) { - 16, 80, 128 => true, - 32, 64 => !self.hasFeature(.fma), - else => unreachable, - }) { - if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {f}", .{ - ty.fmt(pt), - }); - - var sym_buf: ["__fma?".len]u8 = undefined; - break :result try self.genCall(.{ .extern_func = .{ - .return_type = ty.toIntern(), - .param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() }, - .sym = std.fmt.bufPrint(&sym_buf, "{s}fma{s}", .{ - floatLibcAbiPrefix(ty), - floatLibcAbiSuffix(ty), - }) catch unreachable, - } }, &.{ ty, ty, ty }, &.{ - .{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand }, - }, .{}); - } - - var mcvs: [3]MCValue = undefined; - var locks: [3]?RegisterManager.RegisterLock = @splat(null); - defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); - var order: [3]u2 = @splat(0); - var unused: std.StaticBitSet(3) = .initFull(); - for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| { - const op_index: u2 = @intCast(op_i); - mcv.* = try self.resolveInst(op); - if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) { - order[op_index] = 1; - unused.unset(0); - } else if (unused.isSet(2) and mcv.isBase()) { - order[op_index] = 3; - unused.unset(2); - } - switch (mcv.*) { - .register => |reg| lock.* = self.register_manager.lockReg(reg), - else => {}, - } - } - for (&order, &mcvs, &locks) |*mop_index, *mcv, *lock| { - if (mop_index.* != 0) continue; - mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?)); - if (mop_index.* > 1 and mcv.isRegister()) continue; - const reg = try self.copyToTmpRegister(ty, mcv.*); - mcv.* = .{ .register = reg }; - if (lock.*) |old_lock| self.register_manager.unlockReg(old_lock); - lock.* = self.register_manager.lockRegAssumeUnused(reg); - } - - const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or - std.mem.eql(u2, &order, &.{ 3, 1, 2 })) - switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(self.target)) { - 32 => .{ .v_ss, .fmadd132 }, - 64 => .{ .v_sd, .fmadd132 }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (ty.childType(zcu).floatBits(self.target)) { - 32 => switch (ty.vectorLen(zcu)) { - 1 => .{ .v_ss, .fmadd132 }, - 2...8 => .{ .v_ps, .fmadd132 }, - else => null, - }, - 64 => switch (ty.vectorLen(zcu)) { - 1 => .{ .v_sd, .fmadd132 }, - 2...4 => .{ .v_pd, .fmadd132 }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - } - else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 })) - switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(self.target)) { - 32 => .{ .v_ss, .fmadd213 }, - 64 => .{ .v_sd, .fmadd213 }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (ty.childType(zcu).floatBits(self.target)) { - 32 => switch (ty.vectorLen(zcu)) { - 1 => .{ .v_ss, .fmadd213 }, - 2...8 => .{ .v_ps, .fmadd213 }, - else => null, - }, - 64 => switch (ty.vectorLen(zcu)) { - 1 => .{ .v_sd, .fmadd213 }, - 2...4 => .{ .v_pd, .fmadd213 }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - } - else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 })) - switch (ty.zigTypeTag(zcu)) { - .float => switch (ty.floatBits(self.target)) { - 32 => .{ .v_ss, .fmadd231 }, - 64 => .{ .v_sd, .fmadd231 }, - 16, 80, 128 => null, - else => unreachable, - }, - .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { - .float => switch (ty.childType(zcu).floatBits(self.target)) { - 32 => switch (ty.vectorLen(zcu)) { - 1 => .{ .v_ss, .fmadd231 }, - 2...8 => .{ .v_ps, .fmadd231 }, - else => null, - }, - 64 => switch (ty.vectorLen(zcu)) { - 1 => .{ .v_sd, .fmadd231 }, - 2...4 => .{ .v_pd, .fmadd231 }, - else => null, - }, - 16, 80, 128 => null, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - } - else - unreachable) orelse return self.fail("TODO implement airMulAdd for {f}", .{ty.fmt(pt)}); - - var mops: [3]MCValue = undefined; - for (order, mcvs) |mop_index, mcv| mops[mop_index - 1] = mcv; - - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const mop1_reg = registerAlias(mops[0].getReg().?, abi_size); - const mop2_reg = registerAlias(mops[1].getReg().?, abi_size); - if (mops[2].isRegister()) try self.asmRegisterRegisterRegister( - mir_tag, - mop1_reg, - mop2_reg, - registerAlias(mops[2].getReg().?, abi_size), - ) else try self.asmRegisterRegisterMemory( - mir_tag, - mop1_reg, - mop2_reg, - try mops[2].mem(self, .{ .size = .fromSize(abi_size) }), - ); - break :result mops[0]; - }; - return self.finishAir(inst, result, ops); -} - -fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty; - const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque); - - const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) { - .x86_64_sysv => result: { - const info = self.va_info.sysv; - const dst_fi = try self.allocFrameIndex(.initSpill(va_list_ty, zcu)); - var field_off: u31 = 0; - // gp_offset: c_uint, - try self.genSetMem( - .{ .frame = dst_fi }, - field_off, - .c_uint, - .{ .immediate = info.gp_count * 8 }, - .{}, - ); - field_off += @intCast(Type.c_uint.abiSize(zcu)); - // fp_offset: c_uint, - try self.genSetMem( - .{ .frame = dst_fi }, - field_off, - .c_uint, - .{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 }, - .{}, - ); - field_off += @intCast(Type.c_uint.abiSize(zcu)); - // overflow_arg_area: *anyopaque, - try self.genSetMem( - .{ .frame = dst_fi }, - field_off, - ptr_anyopaque_ty, - .{ .lea_frame = info.overflow_arg_area }, - .{}, - ); - field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu)); - // reg_save_area: *anyopaque, - try self.genSetMem( - .{ .frame = dst_fi }, - field_off, - ptr_anyopaque_ty, - .{ .lea_frame = info.reg_save_area }, - .{}, - ); - field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu)); - break :result .{ .load_frame = .{ .index = dst_fi } }; - }, - .x86_64_win => return self.fail("TODO implement c_va_start for Win64", .{}), - else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}), - }; - return self.finishAir(inst, result, .{ .none, .none, .none }); -} - -fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const ty = self.typeOfIndex(inst); - const promote_ty = self.promoteVarArg(ty); - const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque); - const unused = self.liveness.isUnused(inst); - - const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) { - .x86_64_sysv => result: { - try self.spillEflagsIfOccupied(); - - const tmp_regs = - try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); - const offset_reg = tmp_regs[0].to32(); - const addr_reg = tmp_regs[1].to64(); - const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); - defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); - - const promote_mcv = try self.allocTempRegOrMem(promote_ty, true); - const promote_lock = switch (promote_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (promote_lock) |lock| self.register_manager.unlockReg(lock); - - const ptr_arg_list_reg = - try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand }); - const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg); - defer self.register_manager.unlockReg(ptr_arg_list_lock); - - const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } }; - const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } }; - const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } }; - const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } }; - - const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target, .arg), .none); - switch (classes[0]) { - .integer => { - assert(classes.len == 1); - - try self.genSetReg(offset_reg, .c_uint, gp_offset, .{}); - try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u( - abi.SysV.c_abi_int_param_regs.len * 8, - )); - const mem_reloc = try self.asmJccReloc(.ae, undefined); - - try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{}); - if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{ - .base = .{ .reg = addr_reg }, - .mod = .{ .rm = .{ .index = offset_reg.to64() } }, - }); - try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{ - .base = .{ .reg = offset_reg.to64() }, - .mod = .{ .rm = .{ .disp = 8 } }, - }); - try self.genCopy(.c_uint, gp_offset, .{ .register = offset_reg }, .{}); - const done_reloc = try self.asmJmpReloc(undefined); - - self.performReloc(mem_reloc); - try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{}); - try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{ - .base = .{ .reg = addr_reg }, - .mod = .{ .rm = .{ .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)) } }, - }); - try self.genCopy( - ptr_anyopaque_ty, - overflow_arg_area, - .{ .register = offset_reg.to64() }, - .{}, - ); - - self.performReloc(done_reloc); - if (!unused) try self.genCopy(promote_ty, promote_mcv, .{ - .indirect = .{ .reg = addr_reg }, - }, .{}); - }, - .sse => { - assert(classes.len == 1); - - try self.genSetReg(offset_reg, .c_uint, fp_offset, .{}); - try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u( - abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16, - )); - const mem_reloc = try self.asmJccReloc(.ae, undefined); - - try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{}); - if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{ - .base = .{ .reg = addr_reg }, - .mod = .{ .rm = .{ .index = offset_reg.to64() } }, - }); - try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{ - .base = .{ .reg = offset_reg.to64() }, - .mod = .{ .rm = .{ .disp = 16 } }, - }); - try self.genCopy(.c_uint, fp_offset, .{ .register = offset_reg }, .{}); - const done_reloc = try self.asmJmpReloc(undefined); - - self.performReloc(mem_reloc); - try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{}); - try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{ - .base = .{ .reg = addr_reg }, - .mod = .{ .rm = .{ .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)) } }, - }); - try self.genCopy( - ptr_anyopaque_ty, - overflow_arg_area, - .{ .register = offset_reg.to64() }, - .{}, - ); - - self.performReloc(done_reloc); - if (!unused) try self.genCopy(promote_ty, promote_mcv, .{ - .indirect = .{ .reg = addr_reg }, - }, .{}); - }, - .memory => { - assert(classes.len == 1); - unreachable; - }, - else => return self.fail("TODO implement c_va_arg for {f} on SysV", .{promote_ty.fmt(pt)}), - } - - if (unused) break :result .unreach; - if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv; - - if (!promote_ty.isRuntimeFloat()) { - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(ty, dst_mcv, promote_mcv, .{}); - break :result dst_mcv; - } - - assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type); - const dst_mcv = if (promote_mcv.isRegister()) - promote_mcv - else - try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv); - const dst_reg = dst_mcv.getReg().?.to128(); - const dst_lock = self.register_manager.lockReg(dst_reg); - defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - - if (self.hasFeature(.avx)) if (promote_mcv.isBase()) try self.asmRegisterRegisterMemory( - .{ .v_ss, .cvtsd2 }, - dst_reg, - dst_reg, - try promote_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegisterRegister( - .{ .v_ss, .cvtsd2 }, - dst_reg, - dst_reg, - (if (promote_mcv.isRegister()) - promote_mcv.getReg().? - else - try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), - ) else if (promote_mcv.isBase()) try self.asmRegisterMemory( - .{ ._ss, .cvtsd2 }, - dst_reg, - try promote_mcv.mem(self, .{ .size = .qword }), - ) else try self.asmRegisterRegister( - .{ ._ss, .cvtsd2 }, - dst_reg, - (if (promote_mcv.isRegister()) - promote_mcv.getReg().? - else - try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), - ); - break :result promote_mcv; - }, - .x86_64_win => return self.fail("TODO implement c_va_arg for Win64", .{}), - else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}), - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airVaCopy(self: *CodeGen, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const ptr_va_list_ty = self.typeOf(ty_op.operand); - - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand }); - return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); -} - -fn airVaEnd(self: *CodeGen, inst: Air.Inst.Index) !void { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - return self.finishAir(inst, .unreach, .{ un_op, .none, .none }); -} - -fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue { - const zcu = self.pt.zcu; - const ty = self.typeOf(ref); - - // If the type has no codegen bits, no need to store it. - if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none; - - const mcv: MCValue = if (ref.toIndex()) |inst| mcv: { - break :mcv self.inst_tracking.getPtr(inst).?.short; - } else mcv: { - break :mcv try self.lowerValue(.fromInterned(ref.toInterned().?)); - }; - - switch (mcv) { - .none, .unreach, .dead => unreachable, - else => return mcv, - } -} - -fn getResolvedInstValue(self: *CodeGen, inst: Air.Inst.Index) *InstTracking { - const tracking = self.inst_tracking.getPtr(inst).?; - return switch (tracking.short) { - .none, .unreach, .dead => unreachable, - else => tracking, - }; -} - -/// If the MCValue is an immediate, and it does not fit within this type, -/// we put it in a register. -/// A potential opportunity for future optimization here would be keeping track -/// of the fact that the instruction is available both as an immediate -/// and as a register. -fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue { - const mcv = try self.resolveInst(operand); - const ti = @typeInfo(T).int; - switch (mcv) { - .immediate => |imm| { - // This immediate is unsigned. - const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed)); - if (imm >= std.math.maxInt(U)) { - return MCValue{ .register = try self.copyToTmpRegister(.usize, mcv) }; - } - }, - else => {}, - } - return mcv; -} - -fn lowerValue(cg: *CodeGen, val: Value) Allocator.Error!MCValue { - return switch (try codegen.lowerValue(cg.pt, val, cg.target)) { - .none => .none, - .undef => .undef, - .immediate => |imm| .{ .immediate = imm }, - .lea_nav => |nav| .{ .lea_nav = nav }, - .lea_uav => |uav| .{ .lea_uav = uav }, - .load_uav => |uav| .{ .load_uav = uav }, - }; -} - -const CallMCValues = struct { - args: []MCValue, - air_arg_count: u32, - return_value: InstTracking, - stack_byte_count: u31, - stack_align: InternPool.Alignment, - gp_count: u32, - fp_count: u32, - err_ret_trace_reg: Register, - - fn deinit(self: *CallMCValues, func: *CodeGen) void { - func.gpa.free(self.args); - self.* = undefined; - } -}; - -const win64_shadow_space = 8 * 4; - -/// Caller must call `CallMCValues.deinit`. -fn resolveCallingConventionValues( - cg: *CodeGen, - fn_info: InternPool.Key.FuncType, - var_args: []const Type, - stack_frame_base: FrameIndex, -) !CallMCValues { - const pt = cg.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const cc = fn_info.cc; - - const ExpectedContents = extern struct { - param_types: [32][@sizeOf(Type)]u8 align(@alignOf(Type)), - }; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); - const allocator = stack.get(); - - const param_types = try allocator.alloc(Type, fn_info.param_types.len + var_args.len); - defer allocator.free(param_types); - - for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*param_ty, arg_ty| - param_ty.* = .fromInterned(arg_ty); - for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty| - param_ty.* = cg.promoteVarArg(arg_ty); - - var result: CallMCValues = .{ - .args = try cg.gpa.alloc(MCValue, param_types.len), - .air_arg_count = 0, - // These undefined values must be populated before returning from this function. - .return_value = undefined, - .stack_byte_count = 0, - .stack_align = undefined, - .gp_count = 0, - .fp_count = 0, - .err_ret_trace_reg = .none, - }; - errdefer cg.gpa.free(result.args); - - const ret_ty: Type = .fromInterned(fn_info.return_type); - switch (cc) { - .naked => { - assert(result.args.len == 0); - result.return_value = .init(.unreach); - result.stack_align = switch (cg.target.cpu.arch) { - else => unreachable, - .x86 => .@"4", - .x86_64 => .@"8", - }; - }, - .x86_64_sysv, .x86_64_win => |cc_opts| { - const param_gpr = abi.getCAbiIntParamRegs(cc); - var param_gpr_index: u32 = 0; - const param_sse = abi.getCAbiSseParamRegs(cc, cg.target); - var param_sse_index: u32 = 0; - - result.stack_align = .fromByteUnits(cc_opts.incoming_stack_alignment orelse 16); - - switch (cc) { - .x86_64_sysv => {}, - .x86_64_win => result.stack_byte_count += @intCast(win64_shadow_space), - else => unreachable, - } - - // Return values - if (ret_ty.isNoReturn(zcu)) { - result.return_value = .init(.unreach); - } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - // TODO: is this even possible for C calling convention? - result.return_value = .init(.none); - } else { - var ret_tracking: [4]InstTracking = undefined; - var ret_tracking_len: u32 = 0; - var ret_gpr = abi.getCAbiIntReturnRegs(cc); - var ret_sse = abi.getCAbiSseReturnRegs(cc); - var ret_x87 = abi.getCAbiX87ReturnRegs(cc); - - const classes = switch (cc) { - .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, cg.target, .ret), .none), - .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu, cg.target)}, - else => unreachable, - }; - for (classes) |class| switch (class) { - .integer => { - ret_tracking[ret_tracking_len] = .init(.{ .register = registerAlias( - ret_gpr[0], - @intCast(@min(ret_ty.abiSize(zcu), 8)), - ) }); - ret_tracking_len += 1; - ret_gpr = ret_gpr[1..]; - }, - .sse, .float, .float_combine, .win_i128 => { - const abi_size: u32 = @intCast(ret_ty.abiSize(zcu)); - const reg_size = @min(abi_size, cg.vectorSize(.float)); - var byte_offset: u32 = 0; - while (byte_offset < abi_size) : (byte_offset += reg_size) { - const ret_sse_reg = registerAlias(ret_sse[0], reg_size); - ret_sse = ret_sse[1..]; - - ret_tracking[ret_tracking_len] = .init(.{ .register = ret_sse_reg }); - ret_tracking_len += 1; - } - }, - .sseup => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.sse)), - .x87 => { - ret_tracking[ret_tracking_len] = .init(.{ .register = ret_x87[0] }); - ret_tracking_len += 1; - ret_x87 = ret_x87[1..]; - }, - .x87up => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.x87)), - .memory, .integer_per_element => { - ret_tracking[ret_tracking_len] = .{ - .short = .{ .indirect = .{ .reg = ret_gpr[0].to64() } }, - .long = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }, - }; - ret_tracking_len += 1; - ret_gpr = ret_gpr[1..]; - param_gpr_index += 1; - }, - .none => unreachable, - }; - result.return_value = switch (ret_tracking_len) { - else => unreachable, - 1 => ret_tracking[0], - 2 => .init(.{ .register_pair = .{ - ret_tracking[0].short.register, - ret_tracking[1].short.register, - } }), - 3 => .init(.{ .register_triple = .{ - ret_tracking[0].short.register, - ret_tracking[1].short.register, - ret_tracking[2].short.register, - } }), - 4 => .init(.{ .register_quadruple = .{ - ret_tracking[0].short.register, - ret_tracking[1].short.register, - ret_tracking[2].short.register, - ret_tracking[3].short.register, - } }), - }; - } - - // Input params - params: for (param_types, result.args) |ty, *arg| { - assert(ty.hasRuntimeBitsIgnoreComptime(zcu)); - result.air_arg_count += 1; - switch (cc) { - .x86_64_sysv => {}, - .x86_64_win => { - param_gpr_index = @max(param_gpr_index, param_sse_index); - param_sse_index = param_gpr_index; - }, - else => unreachable, - } - - var arg_mcv: [4]MCValue = undefined; - var arg_mcv_len: u32 = 0; - - const classes = switch (cc) { - .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .arg), .none), - .x86_64_win => &.{abi.classifyWindows(ty, zcu, cg.target)}, - else => unreachable, - }; - classes: for (classes) |class| switch (class) { - .integer => { - if (param_gpr_index >= param_gpr.len) break; - arg_mcv[arg_mcv_len] = .{ - .register = registerAlias(param_gpr[param_gpr_index], @intCast(@min(ty.abiSize(zcu), 8))), - }; - arg_mcv_len += 1; - param_gpr_index += 1; - }, - .sse, .float, .float_combine => { - const abi_size: u32 = @intCast(ty.abiSize(zcu)); - const reg_size = @min(abi_size, cg.vectorSize(.float)); - var byte_offset: u32 = 0; - while (byte_offset < abi_size) : (byte_offset += reg_size) { - if (param_sse_index >= param_sse.len) break :classes; - - const param_sse_reg = registerAlias(param_sse[param_sse_index], reg_size); - param_sse_index += 1; - - arg_mcv[arg_mcv_len] = .{ .register = param_sse_reg }; - arg_mcv_len += 1; - } - }, - .sseup => assert(arg_mcv[arg_mcv_len - 1].register.isClass(.sse)), - .x87, .x87up, .memory, .win_i128 => switch (cc) { - .x86_64_sysv => switch (class) { - .x87, .x87up, .memory => break, - else => unreachable, - }, - .x86_64_win => if (ty.abiSize(zcu) > 8) { - if (param_gpr_index < param_gpr.len) { - arg_mcv[arg_mcv_len] = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }; - arg_mcv_len += 1; - param_gpr_index += 1; - } else { - assert(arg_mcv_len == 0); - const param_align = Type.usize.abiAlignment(zcu); - result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); - result.stack_align = result.stack_align.max(param_align); - arg.* = .{ .indirect_load_frame = .{ - .index = stack_frame_base, - .off = result.stack_byte_count, - } }; - result.stack_byte_count += @intCast(Type.usize.abiSize(zcu)); - continue :params; - } - } else break, - else => unreachable, - }, - .none => unreachable, - .integer_per_element => { - const remaining_param_gpr_len: u3 = @intCast(param_gpr.len - param_gpr_index); - param_gpr_index = @intCast(param_gpr.len); - - const frame_elem_align = 8; - const frame_elems_len = ty.vectorLen(zcu) - remaining_param_gpr_len; - const frame_elem_size = std.mem.alignForward(u64, ty.childType(zcu).abiSize(zcu), frame_elem_align); - const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size); - - result.stack_byte_count = std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align); - arg_mcv[arg_mcv_len] = .{ .elementwise_args = .{ - .regs = remaining_param_gpr_len, - .frame_off = @intCast(result.stack_byte_count), - .frame_index = stack_frame_base, - } }; - arg_mcv_len += 1; - result.stack_byte_count += frame_size; - }, - } else { - arg.* = switch (arg_mcv_len) { - else => unreachable, - 1 => arg_mcv[0], - 2 => .{ .register_pair = .{ - arg_mcv[0].register, - arg_mcv[1].register, - } }, - 3 => .{ .register_triple = .{ - arg_mcv[0].register, - arg_mcv[1].register, - arg_mcv[2].register, - } }, - 4 => .{ .register_quadruple = .{ - arg_mcv[0].register, - arg_mcv[1].register, - arg_mcv[2].register, - arg_mcv[3].register, - } }, - }; - continue; - } - - const param_align = ty.abiAlignment(zcu).max(.@"8"); - result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); - result.stack_align = result.stack_align.max(param_align); - arg.* = .{ .load_frame = .{ - .index = stack_frame_base, - .off = result.stack_byte_count, - } }; - result.stack_byte_count += @intCast(ty.abiSize(zcu)); - } - assert(param_gpr_index <= 6); - result.gp_count = param_gpr_index; - assert(param_sse_index <= 16); - result.fp_count = param_sse_index; - }, - .auto => { - result.stack_align = abi.zigcc.stack_align orelse .fromByteUnits(cg.vectorSize(.float)); - - var param_gpr = abi.getCAbiIntParamRegs(cc); - var param_x87 = abi.getCAbiX87ParamRegs(cc); - var param_sse = abi.getCAbiSseParamRegs(cc, cg.target); - - if (zcu.comp.config.any_error_tracing) { - result.err_ret_trace_reg = param_gpr[param_gpr.len - 1]; - param_gpr = param_gpr[0 .. param_gpr.len - 1]; - } - - // Return values - result.return_value = if (ret_ty.isNoReturn(zcu)) - .init(.unreach) - else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) - .init(.none) - else return_value: { - const ret_gpr = abi.getCAbiIntReturnRegs(cc); - const ret_size: u31 = @intCast(ret_ty.abiSize(zcu)); - if (abi.zigcc.return_in_regs) switch (cg.regClassForType(ret_ty)) { - .general_purpose, .gphi => if (ret_size <= @as(u4, switch (cg.target.cpu.arch) { - else => unreachable, - .x86 => 4, - .x86_64 => 8, - })) - break :return_value .init(.{ .register = registerAlias(ret_gpr[0], ret_size) }) - else if (ret_gpr.len >= 2 and ret_ty.isSliceAtRuntime(zcu)) - break :return_value .init(.{ .register_pair = ret_gpr[0..2].* }), - .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87 => if (ret_size <= 16) break :return_value .init(.{ .register = .st0 }), - .sse => if (ret_size <= cg.vectorSize(.float)) break :return_value .init(.{ - .register = registerAlias(abi.getCAbiSseReturnRegs(cc)[0], @max(ret_size, 16)), - }), - }; - const ret_indirect_reg = param_gpr[0]; - param_gpr = param_gpr[1..]; - break :return_value .{ - .short = .{ .indirect = .{ .reg = ret_gpr[0] } }, - .long = .{ .indirect = .{ .reg = ret_indirect_reg } }, - }; - }; - - // Input params - for (param_types, result.args) |param_ty, *arg| { - if (!param_ty.hasRuntimeBitsIgnoreComptime(zcu)) { - arg.* = .none; - continue; - } - result.air_arg_count += 1; - const param_size: u31 = @intCast(param_ty.abiSize(zcu)); - if (abi.zigcc.params_in_regs) switch (cg.regClassForType(param_ty)) { - .general_purpose, .gphi => if (param_gpr.len >= 1 and param_size <= @as(u4, switch (cg.target.cpu.arch) { - else => unreachable, - .x86 => 4, - .x86_64 => 8, - })) { - arg.* = .{ .register = registerAlias(param_gpr[0], param_size) }; - param_gpr = param_gpr[1..]; - continue; - } else if (param_gpr.len >= 2 and param_ty.isSliceAtRuntime(zcu)) { - arg.* = .{ .register_pair = param_gpr[0..2].* }; - param_gpr = param_gpr[2..]; - continue; - }, - .segment, .mmx, .ip, .cr, .dr => unreachable, - .x87 => if (param_x87.len >= 1 and param_size <= 16) { - arg.* = .{ .register = param_x87[0] }; - param_x87 = param_x87[1..]; - continue; - }, - .sse => if (param_sse.len >= 1 and param_size <= cg.vectorSize(.float)) { - arg.* = .{ - .register = registerAlias(param_sse[0], @max(param_size, 16)), - }; - param_sse = param_sse[1..]; - continue; - }, - }; - const param_align = param_ty.abiAlignment(zcu); - result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); - result.stack_align = result.stack_align.max(param_align); - arg.* = .{ .load_frame = .{ - .index = stack_frame_base, - .off = result.stack_byte_count, - } }; - result.stack_byte_count += param_size; - } - }, - else => return cg.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}), - } - - result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count)); - return result; -} - -fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = cg.pt.zcu; - return switch (cg.owner) { - .nav_index => |i| zcu.codegenFail(i, format, args), - .lazy_sym => |s| zcu.codegenFailType(s.ty, format, args), - }; -} - -fn failMsg(cg: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = cg.pt.zcu; - return switch (cg.owner) { - .nav_index => |i| zcu.codegenFailMsg(i, msg), - .lazy_sym => |s| zcu.codegenFailTypeMsg(s.ty, msg), - }; -} - -fn parseRegName(name: []const u8) ?Register { - if (std.mem.startsWith(u8, name, "db")) return @enumFromInt( - @intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null), - ); - return std.meta.stringToEnum(Register, name); -} - -/// Returns register wide enough to hold at least `size_bytes`. -fn registerAlias(reg: Register, size_bytes: u32) Register { - if (size_bytes == 0) unreachable; // should be comptime-known - return switch (reg.class()) { - .general_purpose => if (size_bytes <= 1) - reg.to8() - else if (size_bytes <= 2) - reg.to16() - else if (size_bytes <= 4) - reg.to32() - else if (size_bytes <= 8) - reg.to64() - else - unreachable, - .gphi => if (size_bytes <= 1) - reg - else if (size_bytes <= 2) - reg.to16() - else if (size_bytes <= 4) - reg.to32() - else if (size_bytes <= 8) - reg.to64() - else - unreachable, - .segment => if (size_bytes <= 2) - reg - else - unreachable, - .x87 => if (size_bytes >= 4 and size_bytes <= 16) - reg - else - unreachable, - .mmx => if (size_bytes <= 8) - reg - else - unreachable, - .sse => if (size_bytes <= 16) - reg.to128() - else if (size_bytes <= 32) - reg.to256() - else - unreachable, - .ip => if (size_bytes <= 2) - .ip - else if (size_bytes <= 4) - .eip - else if (size_bytes <= 8) - .rip - else - unreachable, - .cr => if (size_bytes <= 8) - reg - else - unreachable, - .dr => if (size_bytes <= 8) - reg - else - unreachable, - }; -} - -fn memSize(self: *CodeGen, ty: Type) Memory.Size { - const zcu = self.pt.zcu; - return if (self.floatBits(ty)) |float_bits| - .fromBitSize(float_bits) - else if (ty.isVector(zcu) and ty.vectorLen(zcu) == 1 and self.floatBits(ty.childType(zcu)) == 80) - .tbyte - else - .fromSize(@intCast(ty.abiSize(zcu))); -} - -fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type { - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - var parts: [parts_len]Type = undefined; - switch (ip.indexToKey(ty.toIntern())) { - .vector_type => |vector_type| if (std.math.divExact(u32, vector_type.len, parts_len)) |vec_len| return .{ - try pt.vectorType(.{ .len = vec_len, .child = vector_type.child }), - } ** parts_len else |err| switch (err) { - error.DivisionByZero => unreachable, - error.UnexpectedRemainder => {}, - }, - .tuple_type => |tuple_type| if (tuple_type.types.len == parts_len) { - for (&parts, tuple_type.types.get(ip)) |*part, part_ty| part.* = .fromInterned(part_ty); - return parts; - }, - else => {}, - } - const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .other), .none); - if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| { - part.* = switch (class) { - .integer => if (part_i < parts_len - 1) - .u64 - else part: { - const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; - const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8)); - break :part switch (@divExact(ty.abiSize(zcu) - part_i * 8, elem_size)) { - 1 => elem_ty, - else => |array_len| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }), - }; - }, - .float => .f32, - .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }), - .sse => .f64, - else => break, - }; - } else { - var part_sizes: u64 = 0; - for (parts) |part| part_sizes += part.abiSize(zcu); - if (part_sizes == ty.abiSize(zcu)) return parts; - }; - return self.fail("TODO implement splitType({d}, {f})", .{ parts_len, ty.fmt(pt) }); -} - -/// Truncates the value in the register in place. -/// Clobbers any remaining bits. -fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void { - const pt = self.pt; - const zcu = pt.zcu; - const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{ - .signedness = .unsigned, - .bits = @intCast(ty.bitSize(zcu)), - }; - const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return; - try self.spillEflagsIfOccupied(); - switch (int_info.signedness) { - .signed => { - try self.genShiftBinOpMir(.{ ._l, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift }); - try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift }); - }, - .unsigned => { - const mask = ~@as(u64, 0) >> shift; - if (int_info.bits <= 32) { - try self.genBinOpMir(.{ ._, .@"and" }, .u32, .{ .register = reg }, .{ .immediate = mask }); - } else { - const tmp_reg = try self.copyToTmpRegister(.usize, .{ .immediate = mask }); - try self.genBinOpMir(.{ ._, .@"and" }, .usize, .{ .register = reg }, .{ .register = tmp_reg }); - } - }, - } -} - -fn regBitSize(self: *CodeGen, ty: Type) u64 { - const zcu = self.pt.zcu; - const abi_size = ty.abiSize(zcu); - return switch (ty.zigTypeTag(zcu)) { - else => switch (abi_size) { - 1 => 8, - 2 => 16, - 3...4 => 32, - 5...8 => 64, - else => unreachable, - }, - .float => switch (abi_size) { - 1...16 => 128, - 17...32 => 256, - else => unreachable, - }, - }; -} - -fn regExtraBits(self: *CodeGen, ty: Type) u64 { - return self.regBitSize(ty) - ty.bitSize(self.pt.zcu); -} - -fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool { - return switch (feature) { - .@"64bit" => switch (cg.target.cpu.arch) { - else => unreachable, - .x86 => false, - .x86_64 => true, - }, - .false_deps_getmant, - .false_deps_lzcnt_tzcnt, - .false_deps_mulc, - .false_deps_mullq, - .false_deps_perm, - .false_deps_popcnt, - .false_deps_range, - .slow_3ops_lea, - .slow_incdec, - .slow_lea, - .slow_pmaddwd, - .slow_pmulld, - .slow_shld, - .slow_two_mem_ops, - .slow_unaligned_mem_16, - .slow_unaligned_mem_32, - => switch (cg.mod.optimize_mode) { - .Debug, .ReleaseSafe, .ReleaseFast => null, - .ReleaseSmall => false, - }, - .fast_11bytenop, - .fast_15bytenop, - .fast_7bytenop, - .fast_bextr, - .fast_dpwssd, - .fast_gather, - .fast_hops, - .fast_imm16, - .fast_lzcnt, - .fast_movbe, - .fast_scalar_fsqrt, - .fast_scalar_shift_masks, - .fast_shld_rotate, - .fast_variable_crosslane_shuffle, - .fast_variable_perlane_shuffle, - .fast_vector_fsqrt, - .fast_vector_shift_masks, - => switch (cg.mod.optimize_mode) { - .Debug, .ReleaseSafe, .ReleaseFast => null, - .ReleaseSmall => true, - }, - .mmx => false, - .sahf => switch (cg.target.cpu.arch) { - else => unreachable, - .x86 => true, - .x86_64 => null, - }, - else => null, - } orelse cg.target.cpu.has(.x86, feature); -} - -fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type { - const pt = self.pt; - const zcu = pt.zcu; - return self.air.typeOf(inst, &zcu.intern_pool); -} - -fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type { - return Temp.typeOf(.{ .index = inst }, self); -} - -fn intCompilerRtAbiName(int_bits: u32) u8 { - return switch (int_bits) { - 1...32 => 's', - 33...64 => 'd', - 65...128 => 't', - else => unreachable, - }; -} - -fn floatCompilerRtAbiName(float_bits: u32) u8 { - return switch (float_bits) { - 16 => 'h', - 32 => 's', - 64 => 'd', - 80 => 'x', - 128 => 't', - else => unreachable, - }; -} - -fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type { - if (ty.toIntern() == .f16_type and - (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and - self.target.os.tag.isDarwin()) return .u16; - return ty; -} - -fn floatLibcAbiPrefix(ty: Type) []const u8 { - return switch (ty.toIntern()) { - .f16_type, .f80_type => "__", - .f32_type, .f64_type, .f128_type, .c_longdouble_type => "", - else => unreachable, - }; -} - -fn floatLibcAbiSuffix(ty: Type) []const u8 { - return switch (ty.toIntern()) { - .f16_type => "h", - .f32_type => "f", - .f64_type => "", - .f80_type => "x", - .f128_type => "q", - .c_longdouble_type => "l", - else => unreachable, - }; -} - -fn promoteInt(self: *CodeGen, ty: Type) Type { - const pt = self.pt; - const zcu = pt.zcu; - const int_info: InternPool.Key.IntType = switch (ty.toIntern()) { - .bool_type => .{ .signedness = .unsigned, .bits = 1 }, - else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty, - }; - for ([_]Type{ - .c_int, .c_uint, - .c_long, .c_ulong, - .c_longlong, .c_ulonglong, - }) |promote_ty| { - const promote_info = promote_ty.intInfo(zcu); - if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue; - if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and - promote_info.signedness == .signed) <= promote_info.bits) return promote_ty; - } - return ty; -} - -fn promoteVarArg(self: *CodeGen, ty: Type) Type { - if (!ty.isRuntimeFloat()) return self.promoteInt(ty); - switch (ty.floatBits(self.target)) { - 32, 64 => return .f64, - else => |float_bits| { - assert(float_bits == self.target.cTypeBitSize(.longdouble)); - return .c_longdouble; - }, - } -} - -fn unalignedSize(cg: *CodeGen, ty: Type) u64 { - const zcu = cg.pt.zcu; - return switch (zcu.intern_pool.indexToKey(ty.toIntern())) { - .vector_type => |vector_type| switch (vector_type.child) { - .bool_type => ty.abiSize(zcu), - else => Type.fromInterned(vector_type.child).abiSize(zcu) * vector_type.len, - }, - else => ty.abiSize(zcu), - }; -} - -fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 { - const zcu = cg.pt.zcu; - return switch (zcu.intern_pool.indexToKey(ty.toIntern())) { - .vector_type => |vector_type| switch (vector_type.child) { - .bool_type => vector_type.len, - else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)), - }, - else => @intCast(ty.bitSize(zcu)), - }; -} - -fn intInfo(cg: *CodeGen, ty: Type) ?std.builtin.Type.Int { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - var ty_index = ty.ip_index; - while (true) switch (ip.indexToKey(ty_index)) { - .int_type => |int_type| return int_type, - .ptr_type => |ptr_type| return switch (ptr_type.flags.size) { - .one, .many, .c => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() }, - .slice => null, - }, - .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBitsIgnoreComptime(zcu)) - .{ .signedness = .unsigned, .bits = 1 } - else switch (ip.indexToKey(opt_child)) { - .ptr_type => |ptr_type| switch (ptr_type.flags.size) { - .one, .many => switch (ptr_type.flags.is_allowzero) { - false => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() }, - true => null, - }, - .slice, .c => null, - }, - else => null, - }, - .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type) - .hasRuntimeBitsIgnoreComptime(zcu)) .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } else null, - .simple_type => |simple_type| return switch (simple_type) { - .bool => .{ .signedness = .unsigned, .bits = 1 }, - .anyerror => .{ .signedness = .unsigned, .bits = zcu.errorSetBits() }, - .isize => .{ .signedness = .signed, .bits = cg.target.ptrBitWidth() }, - .usize => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() }, - .c_char => .{ .signedness = cg.target.cCharSignedness(), .bits = cg.target.cTypeBitSize(.char) }, - .c_short => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.short) }, - .c_ushort => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.short) }, - .c_int => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.int) }, - .c_uint => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.int) }, - .c_long => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.long) }, - .c_ulong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.long) }, - .c_longlong => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.longlong) }, - .c_ulonglong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.longlong) }, - .f16, .f32, .f64, .f80, .f128, .c_longdouble => null, - .anyopaque, - .void, - .type, - .comptime_int, - .comptime_float, - .noreturn, - .null, - .undefined, - .enum_literal, - .adhoc_inferred_error_set, - .generic_poison, - => unreachable, - }, - .struct_type => { - const loaded_struct = ip.loadStructType(ty_index); - switch (loaded_struct.layout) { - .auto, .@"extern" => return null, - .@"packed" => ty_index = loaded_struct.backingIntTypeUnordered(ip), - } - }, - .union_type => return switch (ip.loadUnionType(ty_index).flagsUnordered(ip).layout) { - .auto, .@"extern" => null, - .@"packed" => .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) }, - }, - .enum_type => ty_index = ip.loadEnumType(ty_index).tag_ty, - .error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = zcu.errorSetBits() }, - else => return null, - }; -} - -fn floatBits(cg: *CodeGen, ty: Type) ?u16 { - return if (ty.isRuntimeFloat()) ty.floatBits(cg.target) else null; -} - -const Temp = struct { - index: Air.Inst.Index, - - fn unwrap(temp: Temp, cg: *CodeGen) union(enum) { - ref: Air.Inst.Ref, - temp: Index, - err_ret_trace, - } { - switch (temp.index.unwrap()) { - .ref => |ref| return .{ .ref = ref }, - .target => |target_index| { - if (temp.index == err_ret_trace_index) return .err_ret_trace; - const temp_index: Index = @enumFromInt(target_index); - assert(temp_index.isValid(cg)); - return .{ .temp = temp_index }; - }, - } - } - - fn typeOf(temp: Temp, cg: *CodeGen) Type { - return switch (temp.unwrap(cg)) { - .ref => switch (cg.air.instructions.items(.tag)[@intFromEnum(temp.index)]) { - .loop_switch_br => cg.typeOf(cg.air.unwrapSwitch(temp.index).operand), - else => cg.air.typeOfIndex(temp.index, &cg.pt.zcu.intern_pool), - }, - .temp => |temp_index| temp_index.typeOf(cg), - .err_ret_trace => .usize, - }; - } - - fn isMut(temp: Temp, cg: *CodeGen) bool { - return switch (temp.unwrap(cg)) { - .ref, .err_ret_trace => false, - .temp => |temp_index| switch (temp_index.tracking(cg).short) { - .none, - .unreach, - .dead, - .undef, - .immediate, - .eflags, - .register_offset, - .register_mask, - .memory, - .indirect, - .indirect_load_frame, - .lea_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .lea_extern_func, - .load_extern_func, - .elementwise_args, - .reserved_frame, - .air_ref, - => false, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - => true, - .load_frame => |frame_addr| !frame_addr.index.isNamed(), - }, - }; - } - - fn tracking(temp: Temp, cg: *CodeGen) InstTracking { - return cg.inst_tracking.get(temp.index).?; - } - - fn getOffset(temp: Temp, off: i32, cg: *CodeGen) InnerError!Temp { - const new_temp_index = cg.next_temp_index; - cg.temp_type[@intFromEnum(new_temp_index)] = .usize; - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - const mcv = temp.tracking(cg).short; - switch (mcv) { - else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), - .register => |reg| { - const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ - .base = .{ .reg = reg.to64() }, - .mod = .{ .rm = .{ .disp = off } }, - }); - }, - .register_offset => |reg_off| { - const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ - .base = .{ .reg = reg_off.reg.to64() }, - .mod = .{ .rm = .{ .disp = reg_off.off + off } }, - }); - }, - .load_frame, .load_nav, .lea_nav, .load_uav, .lea_uav, .load_lazy_sym, .lea_lazy_sym => { - const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{ .reg = new_reg, .off = off } }); - try cg.genSetReg(new_reg, .usize, mcv, .{}); - }, - .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + off, - } }), - } - return .{ .index = new_temp_index.toIndex() }; - } - - fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) InnerError!void { - if (off == 0) return; - switch (temp.unwrap(cg)) { - .ref, .err_ret_trace => {}, - .temp => |temp_index| { - const temp_tracking = temp_index.tracking(cg); - switch (temp_tracking.short) { - else => {}, - .register => |reg| { - try cg.freeValue(temp_tracking.long, .{}); - temp_tracking.* = .init(.{ .register_offset = .{ - .reg = reg, - .off = off, - } }); - return; - }, - .register_offset => |reg_off| { - try cg.freeValue(temp_tracking.long, .{}); - temp_tracking.* = .init(.{ .register_offset = .{ - .reg = reg_off.reg, - .off = reg_off.off + off, - } }); - return; - }, - .lea_frame => |frame_addr| { - assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr)); - temp_tracking.* = .init(.{ .lea_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + off, - } }); - return; - }, - } - }, - } - const new_temp = try temp.getOffset(off, cg); - try temp.die(cg); - temp.* = new_temp; - } - - fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) InnerError!Temp { - const new_temp_index = cg.next_temp_index; - cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty; - switch (temp.tracking(cg).short) { - else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), - .immediate => |imm| { - assert(limb_index == 0); - new_temp_index.tracking(cg).* = .init(.{ .immediate = imm }); - }, - .register => |reg| { - assert(limb_index == 0); - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64()); - }, - .register_pair => |regs| { - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64()); - }, - .register_offset => |reg_off| { - assert(limb_index == 0); - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ - .base = .{ .reg = reg_off.reg.to64() }, - .mod = .{ .rm = .{ .disp = reg_off.off + @as(u31, limb_index) * 8 } }, - }); - }, - .load_frame => |frame_addr| { - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ - .base = .{ .frame = frame_addr.index }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = frame_addr.off + @as(u31, limb_index) * 8, - } }, - }); - }, - .lea_frame => |frame_addr| { - assert(limb_index == 0); - new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr }); - }, - .load_nav => |nav| { - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ - .base = .{ .nav = nav }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = @as(u31, limb_index) * 8, - } }, - }); - }, - .lea_nav => |nav| { - assert(limb_index == 0); - new_temp_index.tracking(cg).* = .init(.{ .lea_nav = nav }); - }, - .load_uav => |uav| { - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ - .base = .{ .uav = uav }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = @as(u31, limb_index) * 8, - } }, - }); - }, - .lea_uav => |uav| { - assert(limb_index == 0); - new_temp_index.tracking(cg).* = .init(.{ .lea_uav = uav }); - }, - .load_lazy_sym => |lazy_sym| { - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ - .base = .{ .lazy_sym = lazy_sym }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = @as(u31, limb_index) * 8, - } }, - }); - }, - .lea_lazy_sym => |lazy_sym| { - assert(limb_index == 0); - new_temp_index.tracking(cg).* = .init(.{ .lea_lazy_sym = lazy_sym }); - }, - .load_extern_func => |extern_func| { - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ - .base = .{ .extern_func = extern_func }, - .mod = .{ .rm = .{ - .size = .qword, - .disp = @as(u31, limb_index) * 8, - } }, - }); - }, - .lea_extern_func => |extern_func| { - assert(limb_index == 0); - new_temp_index.tracking(cg).* = .init(.{ .lea_extern_func = extern_func }); - }, - } - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - return .{ .index = new_temp_index.toIndex() }; - } - - fn getByteLen(temp: *Temp, cg: *CodeGen) Select.Error!Temp { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - const ptr_info = ip.indexToKey(temp.typeOf(cg).toIntern()).ptr_type; - switch (ptr_info.flags.size) { - .one => { - const array_info = ip.indexToKey(ptr_info.child).array_type; - return cg.tempInit(.usize, .{ - .immediate = Type.fromInterned(array_info.child).abiSize(zcu) * array_info.len, - }); - }, - .many, .c => unreachable, - .slice => { - const elem_size = Type.fromInterned(ptr_info.child).abiSize(zcu); - var len = try temp.getLimb(.usize, 1, cg); - while (try len.toRegClass(true, .general_purpose, cg)) {} - const len_reg = len.tracking(cg).short.register.to64(); - if (!std.math.isPowerOfTwo(elem_size)) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - len_reg, - len_reg, - .u(elem_size), - ); - } else if (elem_size > 8) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterImmediate( - .{ ._l, .sh }, - len_reg, - .u(std.math.log2_int(u64, elem_size)), - ); - } else if (elem_size != 1) try cg.asmRegisterMemory(.{ ._, .lea }, len_reg, .{ - .mod = .{ .rm = .{ - .index = len_reg, - .scale = .fromFactor(@intCast(elem_size)), - } }, - }); - return len; - }, - } - } - - fn toLimb(temp: *Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) InnerError!void { - switch (temp.unwrap(cg)) { - .ref => {}, - .temp => |temp_index| { - const temp_tracking = temp_index.tracking(cg); - switch (temp_tracking.short) { - else => {}, - .register, .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => { - assert(limb_index == 0); - cg.temp_type[@intFromEnum(temp_index)] = limb_ty; - return; - }, - .register_pair => |regs| { - switch (temp_tracking.long) { - .none, .reserved_frame => {}, - else => temp_tracking.long = - temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(), - } - for (regs, 0..) |reg, reg_index| if (reg_index != limb_index) - cg.register_manager.freeReg(reg); - temp_tracking.* = .init(.{ .register = regs[limb_index] }); - cg.temp_type[@intFromEnum(temp_index)] = limb_ty; - return; - }, - .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) { - assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr)); - temp_tracking.* = .init(.{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + @as(u31, limb_index) * 8, - } }); - cg.temp_type[@intFromEnum(temp_index)] = limb_ty; - return; - }, - } - }, - .err_ret_trace => unreachable, - } - const new_temp = try temp.getLimb(limb_ty, limb_index, cg); - try temp.die(cg); - temp.* = new_temp; - } - - fn toSlicePtr(temp: *Temp, cg: *CodeGen) InnerError!void { - const temp_ty = temp.typeOf(cg); - if (temp_ty.isSlice(cg.pt.zcu)) try temp.toLimb(temp_ty.slicePtrFieldType(cg.pt.zcu), 0, cg); - } - - fn toSliceLen(temp: *Temp, cg: *CodeGen) InnerError!void { - try temp.toLimb(.usize, 1, cg); - } - - fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) InnerError!bool { - const ty: Type = ty: switch (temp.unwrap(cg)) { - .ref => |ref| cg.typeOf(ref), - .temp => |temp_index| { - const temp_tracking = temp_index.tracking(cg); - if (temp_tracking.short == .register and - temp_tracking.short.register == new_reg) return false; - break :ty temp_index.typeOf(cg); - }, - .err_ret_trace => .usize, - }; - const new_temp_index = cg.next_temp_index; - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - try cg.register_manager.getReg(new_reg, new_temp_index.toIndex()); - cg.temp_type[@intFromEnum(new_temp_index)] = ty; - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - while (try temp.toBase(false, cg)) {} - try temp.readTo(ty, .{ .register = new_reg }, .{}, cg); - try temp.die(cg); - temp.* = .{ .index = new_temp_index.toIndex() }; - return true; - } - - fn toRegPair(temp: *Temp, new_regs: [2]Register, cg: *CodeGen) InnerError!bool { - const ty: Type = ty: switch (temp.unwrap(cg)) { - .ref => |ref| cg.typeOf(ref), - .temp => |temp_index| { - const temp_tracking = temp_index.tracking(cg); - if (temp_tracking.short == .register_pair and - std.mem.eql(Register, &temp_tracking.short.register_pair, &new_regs)) return false; - break :ty temp_index.typeOf(cg); - }, - .err_ret_trace => .usize, - }; - const new_temp_index = cg.next_temp_index; - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - for (new_regs) |new_reg| try cg.register_manager.getReg(new_reg, new_temp_index.toIndex()); - cg.temp_type[@intFromEnum(new_temp_index)] = ty; - new_temp_index.tracking(cg).* = .init(.{ .register_pair = new_regs }); - while (try temp.toBase(false, cg)) {} - for (new_regs, 0..) |new_reg, reg_index| try temp.readTo( - .usize, - .{ .register = new_reg }, - .{ .disp = @intCast(8 * reg_index) }, - cg, - ); - try temp.die(cg); - temp.* = .{ .index = new_temp_index.toIndex() }; - return true; - } - - fn toRegClass(temp: *Temp, mut: bool, rc: Register.Class, cg: *CodeGen) InnerError!bool { - const val = temp.tracking(cg).short; - if (!mut or temp.isMut(cg)) switch (val) { - else => {}, - .register => |reg| if (reg.isClass(rc)) return false, - .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.isClass(rc)) return false, - }; - const ty = temp.typeOf(cg); - const new_temp_index = cg.next_temp_index; - cg.temp_type[@intFromEnum(new_temp_index)] = ty; - const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc)); - try cg.genSetReg(new_reg, ty, val, .{}); - new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); - try temp.die(cg); - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - temp.* = .{ .index = new_temp_index.toIndex() }; - return true; - } - - fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) InnerError!void { - while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| { - if (try part_temp.toRegClass(true, .general_purpose, cg)) break; - } else break; - const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg); - const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg); - const result: MCValue = .{ .register_pair = .{ - first_temp_tracking.short.register.to64(), - second_temp_tracking.short.register.to64(), - } }; - const result_temp_index = cg.next_temp_index; - const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; - assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking)); - assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking)); - result_temp_index.tracking(cg).* = .init(result); - cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; - cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); - first_temp.* = result_temp; - second_temp.* = result_temp; - } - - fn withOverflow(temp: *Temp, overflow_temp: *Temp, cg: *CodeGen) InnerError!void { - const temp_index = temp.unwrap(cg).temp; - const temp_tracking = temp_index.tracking(cg); - const overflow_temp_tracking = overflow_temp.unwrap(cg).temp.tracking(cg); - switch (temp_tracking.short) { - .register => |reg| switch (overflow_temp_tracking.short) { - .eflags => |overflow_cc| { - const result_temp_index = cg.next_temp_index; - const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; - assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); - assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); - result_temp_index.tracking(cg).* = .init(.{ .register_overflow = .{ .reg = reg, .eflags = overflow_cc } }); - cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; - cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); - temp.* = result_temp; - overflow_temp.* = result_temp; - return; - }, - .register => |overflow_reg| { - const result_temp_index = cg.next_temp_index; - const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; - assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); - assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); - result_temp_index.tracking(cg).* = .init(.{ .register_pair = .{ reg, overflow_reg } }); - cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; - cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); - temp.* = result_temp; - overflow_temp.* = result_temp; - return; - }, - else => {}, - }, - .load_frame => { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - const field_tys = ip.indexToKey(temp_index.typeOf(cg).toIntern()).tuple_type.types.get(ip); - try temp.write(overflow_temp, .{ .disp = @intCast(Type.fromInterned(field_tys[0]).abiSize(zcu)) }, cg); - try overflow_temp.die(cg); - overflow_temp.* = temp.*; - return; - }, - else => {}, - } - std.debug.panic("{s}: {f} {f}\n", .{ @src().fn_name, temp_tracking, overflow_temp_tracking }); - } - - fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void { - assert(info.scalar != .none); - const mcv = &temp.unwrap(cg).temp.tracking(cg).short; - const reg = mcv.register; - mcv.* = .{ .register_mask = .{ .reg = reg, .info = info } }; - } - - fn toLea(temp: *Temp, cg: *CodeGen) InnerError!bool { - switch (temp.tracking(cg).short) { - .none, - .unreach, - .dead, - .undef, - .eflags, - .register_pair, - .register_triple, - .register_quadruple, - .register_overflow, - .register_mask, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, // not a valid pointer - .immediate, - .register, - .register_offset, - .lea_frame, - => return false, - .memory, - .indirect, - .load_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => return temp.toRegClass(true, .general_purpose, cg), - } - } - - fn toMemory(temp: *Temp, mut: bool, cg: *CodeGen) InnerError!bool { - const temp_tracking = temp.tracking(cg); - if ((!mut or temp.isMut(cg)) and temp_tracking.short.isMemory()) return false; - const new_temp_index = cg.next_temp_index; - const ty = temp.typeOf(cg); - cg.temp_type[@intFromEnum(new_temp_index)] = ty; - const new_frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu)); - try cg.genSetMem(.{ .frame = new_frame_index }, 0, ty, temp_tracking.short, .{}); - new_temp_index.tracking(cg).* = .init(.{ .load_frame = .{ .index = new_frame_index } }); - try temp.die(cg); - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - temp.* = .{ .index = new_temp_index.toIndex() }; - return true; - } - - // hack around linker relocation bugs - fn toBase(temp: *Temp, mut: bool, cg: *CodeGen) InnerError!bool { - const temp_tracking = temp.tracking(cg); - if ((!mut or temp.isMut(cg)) and temp_tracking.short.isBase()) return false; - if (try temp.toMemory(mut, cg)) return true; - const new_temp_index = cg.next_temp_index; - cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg); - const new_reg = - try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); - try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{}); - new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } }); - try temp.die(cg); - cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); - temp.* = .{ .index = new_temp_index.toIndex() }; - return true; - } - - const AccessOptions = struct { - disp: i32 = 0, - safe: bool = false, - }; - - fn load(ptr: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) InnerError!Temp { - const val = try cg.tempAlloc(val_ty); - try ptr.toOffset(opts.disp, cg); - while (try ptr.toLea(cg)) {} - const val_mcv = val.tracking(cg).short; - switch (val_mcv) { - else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), - .register => |val_reg| try ptr.loadReg(val_ty, registerAlias( - val_reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - ), cg), - inline .register_pair, - .register_triple, - .register_quadruple, - => |val_regs| for (val_regs) |val_reg| { - try ptr.loadReg(val_ty, val_reg, cg); - try ptr.toOffset(@intCast(@divExact(val_reg.size().bitSize(cg.target), 8)), cg); - while (try ptr.toLea(cg)) {} - }, - .register_offset => |val_reg_off| switch (val_reg_off.off) { - 0 => try ptr.loadReg(val_ty, registerAlias( - val_reg_off.reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - ), cg), - else => unreachable, - }, - .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { - var val_ptr = try cg.tempInit(.usize, val_mcv.address()); - var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); - try val_ptr.memcpy(ptr, &len, cg); - try val_ptr.die(cg); - try len.die(cg); - }, - } - return val; - } - - fn store(ptr: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) InnerError!void { - const val_ty = val.typeOf(cg); - try ptr.toOffset(opts.disp, cg); - while (try ptr.toLea(cg)) {} - val_to_gpr: while (true) : (while (try ptr.toLea(cg) or - try val.toRegClass(false, .general_purpose, cg)) - {}) { - const val_mcv = val.tracking(cg).short; - switch (val_mcv) { - else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), - .undef => if (opts.safe) { - var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); - var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); - try ptr.memset(&pat, &len, cg); - try pat.die(cg); - try len.die(cg); - }, - .immediate => |val_imm| { - const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31| - .u(val_uimm31) - else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32| - .s(val_simm32) - else - continue :val_to_gpr; - // hack around linker relocation bugs - switch (ptr.tracking(cg).short) { - else => {}, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, - } - try cg.asmMemoryImmediate( - .{ ._, .mov }, - try ptr.tracking(cg).short.deref().mem(cg, .{ - .size = cg.memSize(val_ty), - }), - val_op, - ); - }, - .eflags => |cc| { - // hack around linker relocation bugs - switch (ptr.tracking(cg).short) { - else => {}, - .lea_nav, - .lea_uav, - .lea_lazy_sym, - .lea_extern_func, - => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, - } - try cg.asmSetccMemory( - cc, - try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }), - ); - }, - .register => |val_reg| try ptr.storeRegs(val_ty, &.{registerAlias( - val_reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - )}, cg), - inline .register_pair, - .register_triple, - .register_quadruple, - => |val_regs| try ptr.storeRegs(val_ty, &val_regs, cg), - .register_offset => |val_reg_off| switch (val_reg_off.off) { - 0 => try ptr.storeRegs(val_ty, &.{registerAlias( - val_reg_off.reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - )}, cg), - else => continue :val_to_gpr, - }, - .register_overflow => |val_reg_ov| { - const ip = &cg.pt.zcu.intern_pool; - const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) { - .tuple_type => |tuple_type| { - const tuple_field_types = tuple_type.types.get(ip); - assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type); - break :first_ty tuple_field_types[0]; - }, - .opt_type => |opt_child| { - assert(!val_ty.optionalReprIsPayload(cg.pt.zcu)); - break :first_ty opt_child; - }, - else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }), - }); - const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu)); - try ptr.storeRegs(first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg); - try ptr.toOffset(first_size, cg); - try cg.asmSetccMemory( - val_reg_ov.eflags, - try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }), - ); - }, - .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr, - .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { - var val_ptr = try cg.tempInit(.usize, val_mcv.address()); - var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); - try ptr.memcpy(&val_ptr, &len, cg); - try val_ptr.die(cg); - try len.die(cg); - }, - } - break; - } - } - - fn read(src: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) InnerError!Temp { - var val = try cg.tempAlloc(val_ty); - while (try src.toBase(false, cg)) {} - try src.readTo(val_ty, val.tracking(cg).short, opts, cg); - return val; - } - - fn readTo(src: *Temp, val_ty: Type, val_mcv: MCValue, opts: AccessOptions, cg: *CodeGen) InnerError!void { - switch (val_mcv) { - else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), - .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias( - val_reg, - @intCast(cg.unalignedSize(val_ty)), - ), cg), - inline .register_pair, .register_triple, .register_quadruple => |val_regs| { - var disp = opts.disp; - for (val_regs) |val_reg| { - try src.readReg(disp, val_ty, val_reg, cg); - disp += @intCast(@divExact(val_reg.size().bitSize(cg.target), 8)); - } - }, - .register_offset => |val_reg_off| switch (val_reg_off.off) { - 0 => try src.readReg(opts.disp, val_ty, registerAlias( - val_reg_off.reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - ), cg), - else => unreachable, - }, - .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { - var val_ptr = try cg.tempInit(.usize, val_mcv.address()); - var src_ptr = - try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp)); - var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); - try val_ptr.memcpy(&src_ptr, &len, cg); - try val_ptr.die(cg); - try src_ptr.die(cg); - try len.die(cg); - }, - } - } - - fn write(dst: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) InnerError!void { - const val_ty = val.typeOf(cg); - while (try dst.toBase(false, cg)) {} - val_to_gpr: while (true) : (while (try dst.toBase(false, cg) or - try val.toRegClass(false, .general_purpose, cg)) - {}) { - const val_mcv = val.tracking(cg).short; - switch (val_mcv) { - else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), - .none => {}, - .undef => if (opts.safe) { - var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); - var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); - var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); - try dst_ptr.memset(&pat, &len, cg); - try dst_ptr.die(cg); - try pat.die(cg); - try len.die(cg); - }, - .immediate => |val_imm| { - const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31| - .u(val_uimm31) - else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32| - .s(val_simm32) - else - continue :val_to_gpr; - try cg.asmMemoryImmediate( - .{ ._, .mov }, - try dst.tracking(cg).short.mem(cg, .{ - .size = cg.memSize(val_ty), - .disp = opts.disp, - }), - val_op, - ); - }, - .eflags => |cc| try cg.asmSetccMemory( - cc, - try dst.tracking(cg).short.mem(cg, .{ - .size = .byte, - .disp = opts.disp, - }), - ), - .register => |val_reg| try dst.writeReg(opts.disp, val_ty, registerAlias( - val_reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - ), cg), - inline .register_pair, - .register_triple, - .register_quadruple, - => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg), - .register_offset => |val_reg_off| switch (val_reg_off.off) { - 0 => try dst.writeReg(opts.disp, val_ty, registerAlias( - val_reg_off.reg, - @intCast(val_ty.abiSize(cg.pt.zcu)), - ), cg), - else => continue :val_to_gpr, - }, - .register_overflow => |val_reg_ov| { - const ip = &cg.pt.zcu.intern_pool; - const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) { - .tuple_type => |tuple_type| { - const tuple_field_types = tuple_type.types.get(ip); - assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type); - break :first_ty tuple_field_types[0]; - }, - .opt_type => |opt_child| { - assert(!val_ty.optionalReprIsPayload(cg.pt.zcu)); - break :first_ty opt_child; - }, - else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }), - }); - const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu)); - try dst.writeReg(opts.disp, first_ty, registerAlias(val_reg_ov.reg, first_size), cg); - try cg.asmSetccMemory( - val_reg_ov.eflags, - try dst.tracking(cg).short.mem(cg, .{ - .size = .byte, - .disp = opts.disp + first_size, - }), - ); - }, - .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr, - .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { - var dst_ptr = - try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); - var val_ptr = try cg.tempInit(.usize, val_mcv.address()); - var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); - try dst_ptr.memcpy(&val_ptr, &len, cg); - try dst_ptr.die(cg); - try val_ptr.die(cg); - try len.die(cg); - }, - } - break; - } - } - - fn loadReg(ptr: *Temp, dst_ty: Type, dst_reg: Register, cg: *CodeGen) InnerError!void { - const dst_rc = dst_reg.class(); - const strat = try cg.moveStrategy(dst_ty, dst_rc, false); - // hack around linker relocation bugs - switch (ptr.tracking(cg).short) { - else => {}, - .lea_nav, .lea_uav, .lea_lazy_sym => if (dst_rc != .general_purpose) - while (try ptr.toRegClass(false, .general_purpose, cg)) {}, - } - try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{ - .size = cg.memSize(dst_ty), - })); - } - - fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { - const zcu = cg.pt.zcu; - const ip = &zcu.intern_pool; - var part_disp: u31 = 0; - var deferred_disp: u31 = 0; - var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu)); - for (src_regs, 0..) |src_reg, part_index| { - const part_ty: Type = if (src_regs.len == 1) - src_ty - else if (cg.intInfo(src_ty)) |int_info| part_ty: { - assert(src_regs.len == std.math.divCeil(u16, int_info.bits, 64) catch unreachable); - break :part_ty .u64; - } else part_ty: switch (ip.indexToKey(src_ty.toIntern())) { - else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, src_ty.fmt(cg.pt) }), - .ptr_type => |ptr_info| { - assert(ptr_info.flags.size == .slice); - assert(src_regs.len == 2); - break :part_ty .usize; - }, - .array_type => { - assert(src_regs.len - part_index == std.math.divCeil(u32, src_abi_size, 8) catch unreachable); - break :part_ty try cg.pt.intType(.unsigned, @as(u16, 8) * @min(src_abi_size, 8)); - }, - .opt_type => |opt_child| switch (ip.indexToKey(opt_child)) { - else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, src_ty.fmt(cg.pt) }), - .ptr_type => |ptr_info| { - assert(ptr_info.flags.size == .slice); - assert(src_regs.len == 2); - break :part_ty .usize; - }, - }, - .struct_type => { - assert(src_regs.len - part_index == std.math.divCeil(u32, src_abi_size, 8) catch unreachable); - break :part_ty switch (src_abi_size) { - 0, 3, 5...7 => unreachable, - 1 => .u8, - 2 => .u16, - 4 => .u32, - else => .u64, - }; - }, - .tuple_type => |tuple_type| { - assert(tuple_type.types.len == src_regs.len); - break :part_ty .fromInterned(tuple_type.types.get(ip)[part_index]); - }, - }; - const part_size: u31 = @intCast(part_ty.abiSize(zcu)); - const src_rc = src_reg.class(); - if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) { - // hack around linker relocation bugs - switch (ptr.tracking(cg).short) { - else => {}, - .lea_nav, .lea_uav, .lea_lazy_sym => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, - } - const strat = try cg.moveStrategy(part_ty, src_rc, false); - try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{ - .size = switch (src_rc) { - else => .fromBitSize(8 * part_size), - .x87 => switch (abi.classifySystemV(src_ty, zcu, cg.target, .other)[part_index]) { - else => unreachable, - .float => .dword, - .float_combine, .sse => .qword, - .x87 => .tbyte, - }, - }, - .disp = part_disp, - }), registerAlias(src_reg, part_size)); - } else { - const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size); - const frame_index = try cg.allocFrameIndex(.init(.{ - .size = frame_size, - .alignment = .fromNonzeroByteUnits(frame_size), - })); - const strat = try cg.moveStrategy(part_ty, src_rc, true); - try strat.write(cg, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, - }, registerAlias(src_reg, frame_size)); - try ptr.toOffset(deferred_disp, cg); - deferred_disp = 0; - var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } }); - var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size }); - try ptr.memcpy(&src_ptr, &len, cg); - try src_ptr.die(cg); - try len.die(cg); - } - part_disp += part_size; - deferred_disp += part_size; - src_abi_size -= part_size; - } - } - - fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) InnerError!void { - const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false); - try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{ - .size = cg.memSize(dst_ty), - .disp = disp, - })); - } - - fn writeReg(dst: Temp, disp: i32, src_ty: Type, src_reg: Register, cg: *CodeGen) InnerError!void { - const src_abi_size: u31 = @intCast(src_ty.abiSize(cg.pt.zcu)); - const src_rc = src_reg.class(); - if (src_rc == .x87 or std.math.isPowerOfTwo(src_abi_size)) { - const strat = try cg.moveStrategy(src_ty, src_rc, false); - try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{ - .size = cg.memSize(src_ty), - .disp = disp, - }), registerAlias(src_reg, src_abi_size)); - } else { - const frame_size = std.math.ceilPowerOfTwoAssert(u32, src_abi_size); - const frame_index = try cg.allocFrameIndex(.init(.{ - .size = frame_size, - .alignment = .fromNonzeroByteUnits(frame_size), - })); - const strat = try cg.moveStrategy(src_ty, src_rc, true); - try strat.write(cg, .{ - .base = .{ .frame = frame_index }, - .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, - }, registerAlias(src_reg, frame_size)); - var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address()); - try dst_ptr.toOffset(disp, cg); - var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } }); - var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size }); - try dst_ptr.memcpy(&src_ptr, &len, cg); - try dst_ptr.die(cg); - try src_ptr.die(cg); - try len.die(cg); - } - } - - fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { - const zcu = cg.pt.zcu; - const classes = std.mem.sliceTo(&abi.classifySystemV(src_ty, zcu, cg.target, .other), .none); - var next_class_index: u4 = 0; - var part_disp = disp; - var remaining_abi_size = src_ty.abiSize(zcu); - for (src_regs) |src_reg| { - const class_index = next_class_index; - const class = classes[class_index]; - next_class_index = @intCast(switch (class) { - .integer, .memory, .float, .float_combine => class_index + 1, - .sse => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.sseup}) orelse classes.len, - .x87 => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.x87up}) orelse classes.len, - .sseup, .x87up, .none, .win_i128, .integer_per_element => unreachable, - }); - const part_size = switch (class) { - .integer, .sse, .memory => @min(8 * @as(u7, next_class_index - class_index), remaining_abi_size), - .x87 => 16, - .float => 4, - .float_combine => 8, - .sseup, .x87up, .none, .win_i128, .integer_per_element => unreachable, - }; - try dst.writeReg(part_disp, switch (class) { - .integer => .u64, - .sse => switch (part_size) { - else => unreachable, - 8 => .f64, - 16 => .vector_2_f64, - 32 => .vector_4_f64, - }, - .x87 => .f80, - .float => .f32, - .float_combine => .vector_2_f32, - .sseup, .x87up, .memory, .none, .win_i128, .integer_per_element => unreachable, - }, src_reg, cg); - part_disp += part_size; - remaining_abi_size -= part_size; - } - assert(next_class_index == classes.len); - } - - fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void { - while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| { - if (try temp.toReg(reg, cg)) break; - } else break; - try cg.asmOpOnly(.{ .@"rep _sb", .mov }); - } - - fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) InnerError!void { - while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) |temp, reg| { - if (try temp.toReg(reg, cg)) break; - } else break; - try cg.asmOpOnly(.{ .@"rep _sb", .sto }); - } - - fn wrapInt(temp: *Temp, cg: *CodeGen) Select.Error!void { - var ops: [1]Temp = .{temp.*}; - var res: [1]Temp = undefined; - try cg.select(&res, &.{temp.typeOf(cg)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._l, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .none, .none } }, - .{ .src = .{ .to_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .bzhi, .dst0q, .src0q, .tmp0q, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_gpr, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 8 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 8 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_b, .add, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_b, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_b, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 16 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 16 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .sll, .dst0x, .src0x, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_w, .sll, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .sll, .dst0y, .src0y, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .dst0y, .dst0y, .uia(16, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sll, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_w, .sll, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_w, .sra, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_w, .sll, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_w, .sra, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .fast_imm16, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .sll, .dst0x, .src0x, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, .p_d, .sll, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, .vp_d, .sll, .dst0y, .src0y, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .dst0y, .dst0y, .uia(32, .dst0, .sub_bit_size), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sll, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .vp_d, .sll, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .vp_d, .sra, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_d, .sll, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, .p_d, .sra, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .bmi2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), .tmp1d, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_q, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .p_q, .add, .dst0x, .lea(.tmp0x), ._, ._ }, - .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, - .{ ._, .vp_q, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .p_q, .add, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, - .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, - .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, - .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .bmi2, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, - .{ .@"0:", ._, .bzhi, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), .tmp1q, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, - .{ ._, ._, .@"and", .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 128 } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .none, .none } }, - .{ .src = .{ .to_mut_sse, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .scalar_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .@"and", .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .si(0), ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, - .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, - .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .none, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, - .{ .@"0:", ._, .@"and", .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, - .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }); - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - temp.* = res[0]; - } - - /// Supports any `op` using `cg.intInfo(lhs.typeOf(cg)).?.signedness` as the signedness. - /// Returns `error.SelectFailed` when `cg.intInfo(lhs.typeOf(cg)) == null`. - fn cmpInts(lhs: *Temp, op: std.math.CompareOperator, rhs: *Temp, cg: *CodeGen) Select.Error!Temp { - var ops: [2]Temp = .{ lhs.*, rhs.* }; - var res: [1]Temp = undefined; - switch (op) { - .lt, .lte, .gte, .gt => { - const commute = switch (op) { - .lt, .gte => false, - .lte, .gt => true, - else => unreachable, - }; - if (commute) std.mem.swap(Temp, &ops[0], &ops[1]); - try cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .g }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm8, .none } }, - .{ .src = .{ .to_gpr, .imm8, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .l }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .a }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm8, .none } }, - .{ .src = .{ .to_gpr, .imm8, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .b }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .g }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm16, .none } }, - .{ .src = .{ .to_gpr, .imm16, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .l }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .a }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm16, .none } }, - .{ .src = .{ .to_gpr, .imm16, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .b }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .g }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm32, .none } }, - .{ .src = .{ .to_gpr, .imm32, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .l }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .a }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm32, .none } }, - .{ .src = .{ .to_gpr, .imm32, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .b }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .g }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .simm32, .none } }, - .{ .src = .{ .to_gpr, .simm32, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .l }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - }, - .dst_temps = .{ .{ .cc = .a }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .simm32, .none } }, - .{ .src = .{ .to_gpr, .simm32, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .b }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .l }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .b }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .l }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_4), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"4", .tmp0, .add_size, -4), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"4", .tmp0, .add_size, -4), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -4), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -4), ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .b }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ }, - .{ ._, ._c, .cl, ._, ._, ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"4", .tmp0, .add_size), ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } }); - if (commute) std.mem.swap(Temp, &ops[0], &ops[1]); - }, - .eq, .neq => { - try cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm8, .none } }, - .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm8, .none } }, - .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm16, .none } }, - .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm16, .none } }, - .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .imm32, .none } }, - .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .imm32, .none } }, - .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .simm32, .none } }, - .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .simm32, .none } }, - .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .cc = .e }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_mm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_mm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_mm, .to_mm, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ }, - .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ }, - .{ ._, ._, .cmp, .tmp0b, .si(-1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_xmm, .to_xmm, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ }, - .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, - .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .fast_imm16, null, null }, - .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ }, - .{ ._, ._, .cmp, .tmp0w, .si(-1), ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_xmm, .mem, .none } }, - .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, - .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ }, - .{ ._, ._, .xor, .tmp0d, .si(std.math.maxInt(u16)), ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .mem, .none } }, - .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_ymm, .to_ymm, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ }, - .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_ymm, .mem, .none } }, - .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, - .{ .src = .{ .to_ymm, .to_ymm, .none } }, - }, - .extra_temps = .{ - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ }, - .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .yword, .is = .xword } }, - .{ .remainder_int = .{ .of = .yword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .memiad(.src1y, .tmp0, .add_size, -16), ._ }, - .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .tmp1x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .memad(.src1x, .add_size, -16), ._ }, - .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .yword, .is = .xword } }, - .{ .remainder_int = .{ .of = .yword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .movu, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ }, - .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .memiad(.src1y, .tmp0, .add_size, -16), ._ }, - .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - .{ ._, .v_pd, .mova, .tmp1x, .memad(.src0x, .add_size, -16), ._, ._ }, - .{ ._, .v_pd, .xor, .tmp1x, .tmp1x, .memad(.src1x, .add_size, -16), ._ }, - .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .yword, .is = .yword } }, - .{ .remainder_int = .{ .of = .yword, .is = .yword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .yword, .is = .yword } }, - .{ .remainder_int = .{ .of = .yword, .is = .yword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, - .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, - .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, - .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .xor, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, .fast_imm16, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp1x, ._, ._ }, - .{ ._, ._, .cmp, .tmp2w, .si(-1), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .{ .remainder_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .sse } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp1x, ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .si(std.math.maxInt(u16)), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, .mmx, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .rc = .mmx } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .cmpeq, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, .p_b, .movmsk, .tmp2d, .tmp1q, ._, ._ }, - .{ ._, ._, .cmp, .tmp2b, .si(-1), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .{ .remainder_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .remainder_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .cc = .z }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._, .xor, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - } }); - }, - } - if (switch (op) { - .lt, .gt, .eq => false, - .lte, .gte, .neq => true, - }) { - const cc = &res[0].unwrap(cg).temp.tracking(cg).short.eflags; - cc.* = cc.negate(); - } - lhs.*, rhs.* = ops; - return res[0]; - } - - fn divTruncInts(lhs: *Temp, rhs: *Temp, cg: *CodeGen) Select.Error!Temp { - var ops: [2]Temp = .{ lhs.*, rhs.* }; - var res: [1]Temp = undefined; - try cg.select(&res, &.{lhs.typeOf(cg)}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .exact_signed_int = 1 }, .{ .exact_signed_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .any, .any, .none } }, - }, - .dst_temps = .{ .{ .imm = 0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .{ .exact_unsigned_int = 1 }, .any }, - .patterns = &.{ - .{ .src = .{ .mut_mem, .any, .none } }, - .{ .src = .{ .to_mut_gpr, .any, .none } }, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .each = .{ .once = &.{} }, - }, .{ - .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .al }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, - .{ ._, .i_, .div, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, - .patterns = &.{ - .{ .src = .{ .mem, .mem, .none } }, - .{ .src = .{ .to_gpr, .mem, .none } }, - .{ .src = .{ .mem, .to_gpr, .none } }, - .{ .src = .{ .to_gpr, .to_gpr, .none } }, - }, - .dst_temps = .{ .{ .reg = .al }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, - .{ ._, ._, .div, .src1b, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1w, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .div, .src1w, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1d, ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, - .{ ._, ._, .div, .src1d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .src1q, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, - .patterns = &.{ - .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, - .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ref = .src0 }, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .xor, .tmp0q, .tmp0q, ._, ._ }, - .{ ._, ._, .div, .src1q, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, - .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, - .none, - } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, - .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, - .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, - .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .slow_incdec, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .reg = .al } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, - .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, - .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i16, .kind = .{ .reg = .ax } }, - .{ .type = .i16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cwd, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, - .{ .type = .u16, .kind = .{ .reg = .dx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i32, .kind = .{ .reg = .eax } }, - .{ .type = .i32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cdq, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .reg = .eax } }, - .{ .type = .u32, .kind = .{ .reg = .edx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .i64, .kind = .{ .reg = .rax } }, - .{ .type = .i64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .cqo, ._, ._, ._, ._ }, - .{ ._, .i_, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .reg = .rax } }, - .{ .type = .u64, .kind = .{ .reg = .rdx } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, - .{ ._, ._, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, - .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .sysv64, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, - .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, - .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, - .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, - .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_cc_abi = .win64, - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mem, .to_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, - .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, - .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__divei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ - .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, - .any, - }, - .patterns = &.{ - .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, - }, - .call_frame = .{ .alignment = .@"16" }, - .extra_temps = .{ - .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, - .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, - .{ .type = .usize, .kind = .{ .extern_func = "__udivei4" } }, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{ .mem, .unused }, - .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, - .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, - .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, - .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, - .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, - .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, - } }, - } }); - lhs.*, rhs.* = ops; - return res[0]; - } - - fn finish( - temp: Temp, - inst: Air.Inst.Index, - op_refs: []const Air.Inst.Ref, - op_temps: []const Temp, - cg: *CodeGen, - ) InnerError!void { - const tomb_bits = cg.liveness.getTombBits(inst); - for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { - if (op_temp.index == temp.index) continue; - if (op_temp.tracking(cg).short != .dead) try op_temp.die(cg); - if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; - if (cg.reused_operands.isSet(op_index)) continue; - try cg.processDeath(op_ref.toIndexAllowNone() orelse continue, .{}); - } - if (cg.liveness.isUnused(inst)) try temp.die(cg) else { - switch (temp.unwrap(cg)) { - .ref, .err_ret_trace => { - const temp_mcv = temp.tracking(cg).short; - const result = result: switch (temp_mcv) { - .none, - .unreach, - .dead, - .indirect_load_frame, - .elementwise_args, - .reserved_frame, - .air_ref, - => unreachable, - .undef, .immediate, .lea_frame => temp_mcv, - .eflags, - .register, - .register_pair, - .register_triple, - .register_quadruple, - .register_offset, - .register_overflow, - .register_mask, - .memory, - .indirect, - .load_frame, - .load_nav, - .lea_nav, - .load_uav, - .lea_uav, - .load_lazy_sym, - .lea_lazy_sym, - .load_extern_func, - .lea_extern_func, - => { - const result = try cg.allocRegOrMem(inst, true); - try cg.genCopy(cg.typeOfIndex(inst), result, temp_mcv, .{}); - break :result result; - }, - }; - tracking_log.debug("{f} => {f} (birth)", .{ inst, result }); - cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result)); - }, - .temp => |temp_index| { - const temp_tracking = temp_index.tracking(cg); - tracking_log.debug("{f} => {f} (birth)", .{ inst, temp_tracking.short }); - cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(temp_tracking.short)); - assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking)); - }, - } - } - for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { - if (op_temp.index != temp.index) continue; - if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; - if (cg.reused_operands.isSet(op_index)) continue; - try cg.processDeath(op_ref.toIndexAllowNone() orelse continue, .{}); - } - } - - fn die(temp: Temp, cg: *CodeGen) InnerError!void { - switch (temp.unwrap(cg)) { - .ref, .err_ret_trace => {}, - .temp => |temp_index| try temp_index.tracking(cg).die(cg, temp_index.toIndex(), .{}), - } - } - - const Index = enum(u5) { - _, - - fn toIndex(index: Index) Air.Inst.Index { - return .fromTargetIndex(@intFromEnum(index)); - } - - fn fromIndex(index: Air.Inst.Index) Index { - return @enumFromInt(index.toTargetIndex()); - } - - fn tracking(index: Index, cg: *CodeGen) *InstTracking { - return &cg.inst_tracking.values()[@intFromEnum(index)]; - } - - fn isValid(index: Index, cg: *CodeGen) bool { - return @intFromEnum(index) < @intFromEnum(cg.next_temp_index) and - index.tracking(cg).short != .dead; - } - - fn typeOf(index: Index, cg: *CodeGen) Type { - assert(index.isValid(cg)); - return cg.temp_type[@intFromEnum(index)]; - } - - const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type); - const Set = std.StaticBitSet(max); - const SafetySet = if (std.debug.runtime_safety) Set else struct { - inline fn initEmpty() @This() { - return .{}; - } - - inline fn isSet(_: @This(), index: usize) bool { - assert(index < max); - return true; - } - - inline fn set(_: @This(), index: usize) void { - assert(index < max); - } - - inline fn eql(_: @This(), _: @This()) bool { - return true; - } - }; - }; -}; - -fn resetTemps(cg: *CodeGen, from_index: Temp.Index) InnerError!void { - var any_valid = false; - for (@intFromEnum(from_index)..@intFromEnum(cg.next_temp_index)) |temp_index| { - const temp: Temp.Index = @enumFromInt(temp_index); - if (temp.isValid(cg)) { - any_valid = true; - tracking_log.err("failed to kill {f}: {f}", .{ - temp.toIndex(), - cg.temp_type[temp_index].fmt(cg.pt), - }); - } - cg.temp_type[temp_index] = undefined; - } - if (any_valid) return cg.fail("failed to kill all temps", .{}); - cg.next_temp_index = from_index; -} - -fn reuseTemp( - cg: *CodeGen, - new_inst: Air.Inst.Index, - old_inst: Air.Inst.Index, - tracking: *InstTracking, -) bool { - switch (tracking.short) { - .register, - .register_pair, - .register_offset, - .register_overflow, - .register_mask, - .indirect, - => for (tracking.short.getRegs()) |tracked_reg| { - if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { - cg.register_manager.registers[tracked_index] = new_inst; - } - }, - .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, - else => {}, - } - switch (tracking.short) { - .eflags, .register_overflow => cg.eflags_inst = new_inst, - else => {}, - } - tracking.reuse(cg, new_inst, old_inst); - return true; -} - -fn tempAlloc(cg: *CodeGen, ty: Type) InnerError!Temp { - const temp_index = cg.next_temp_index; - temp_index.tracking(cg).* = .init( - try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true), - ); - cg.temp_type[@intFromEnum(temp_index)] = ty; - cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); - return .{ .index = temp_index.toIndex() }; -} - -fn tempAllocReg(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) InnerError!Temp { - const temp_index = cg.next_temp_index; - temp_index.tracking(cg).* = .init( - .{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), rs) }, - ); - cg.temp_type[@intFromEnum(temp_index)] = ty; - cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); - return .{ .index = temp_index.toIndex() }; -} - -fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) InnerError!Temp { - const temp_index = cg.next_temp_index; - temp_index.tracking(cg).* = .init( - .{ .register_pair = try cg.register_manager.allocRegs(2, @splat(temp_index.toIndex()), rs) }, - ); - cg.temp_type[@intFromEnum(temp_index)] = ty; - cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); - return .{ .index = temp_index.toIndex() }; -} - -fn tempAllocMem(cg: *CodeGen, ty: Type) InnerError!Temp { - const temp_index = cg.next_temp_index; - temp_index.tracking(cg).* = .init( - try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false), - ); - cg.temp_type[@intFromEnum(temp_index)] = ty; - cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); - return .{ .index = temp_index.toIndex() }; -} - -fn tempInit(cg: *CodeGen, ty: Type, value: MCValue) InnerError!Temp { - const temp_index = cg.next_temp_index; - temp_index.tracking(cg).* = .init(value); - cg.temp_type[@intFromEnum(temp_index)] = ty; - try cg.getValue(value, temp_index.toIndex()); - cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); - return .{ .index = temp_index.toIndex() }; -} - -fn tempFromValue(cg: *CodeGen, value: Value) InnerError!Temp { - return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerValue(value)); -} - -fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp { - return cg.tempMemFromAlignedValue(.none, value); -} - -fn tempMemFromAlignedValue(cg: *CodeGen, alignment: InternPool.Alignment, value: Value) InnerError!Temp { - const ty = value.typeOf(cg.pt.zcu); - return cg.tempInit(ty, .{ .load_uav = .{ - .val = value.toIntern(), - .orig_ty = (try cg.pt.ptrType(.{ - .child = ty.toIntern(), - .flags = .{ - .is_const = true, - .alignment = alignment, - }, - })).toIntern(), - } }); -} - -fn tempFromOperand(cg: *CodeGen, op_ref: Air.Inst.Ref, op_dies: bool) InnerError!Temp { - if (op_dies) { - const temp_index = cg.next_temp_index; - const temp: Temp = .{ .index = temp_index.toIndex() }; - const op_inst = op_ref.toIndex().?; - const tracking = cg.getResolvedInstValue(op_inst); - temp_index.tracking(cg).* = tracking.*; - if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; - cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); - cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); - return temp; - } - - if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; - return cg.tempFromValue(.fromInterned(op_ref.toInterned().?)); -} - -fn tempsFromOperandsInner( - cg: *CodeGen, - inst: Air.Inst.Index, - op_temps: []Temp, - op_refs: []const Air.Inst.Ref, -) InnerError!void { - for (op_temps, 0.., op_refs) |*op_temp, op_index, op_ref| op_temp.* = try cg.tempFromOperand(op_ref, for (op_refs[0..op_index]) |prev_op_ref| { - if (op_ref == prev_op_ref) break false; - } else cg.liveness.operandDies(inst, @intCast(op_index))); -} - -inline fn tempsFromOperands( - cg: *CodeGen, - inst: Air.Inst.Index, - op_refs: anytype, -) InnerError![op_refs.len]Temp { - var op_temps: [op_refs.len]Temp = undefined; - try cg.tempsFromOperandsInner(inst, &op_temps, &op_refs); - return op_temps; -} - -const Operand = union(enum) { - none, - reg: Register, - mem: Memory, - imm: Immediate, - inst: Mir.Inst.Index, -}; - -const Select = struct { - cg: *CodeGen, - types: [@intFromEnum(Select.Operand.Ref.none)]Type, - temps: [@intFromEnum(Select.Operand.Ref.none)]Temp, - labels: [@intFromEnum(Label._)]struct { - backward: ?Mir.Inst.Index, - forward: [3]?Mir.Inst.Index, - }, - top: u3, - - const Error = InnerError || error{SelectFailed}; - - fn emitLabel(s: *Select, label_index: Label) void { - assert(@intFromEnum(label_index) < @intFromEnum(Label._)); - const label = &s.labels[@intFromEnum(label_index)]; - for (&label.forward) |*reloc| { - if (reloc.*) |r| s.cg.performReloc(r); - reloc.* = null; - } - label.backward = @intCast(s.cg.mir_instructions.len); - } - - fn emit(s: *Select, inst: Instruction) InnerError!void { - const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] }; - pseudo: { - switch (inst[0]) { - .@"0:", .@"1:", .@"2:", .@"3:", .@"4:" => |label| s.emitLabel(label), - ._ => {}, - .pseudo => break :pseudo, - } - var mir_ops: [4]CodeGen.Operand = undefined; - inline for (&mir_ops, 3..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s); - s.cg.asmOps(mir_tag, mir_ops) catch |err| switch (err) { - error.InvalidInstruction => { - const fixes = @tagName(mir_tag[0]); - const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; - return s.cg.fail("invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'", .{ - fixes[0..fixes_blank], - @tagName(mir_tag[1]), - fixes[fixes_blank + 1 ..], - @tagName(mir_ops[0]), - @tagName(mir_ops[1]), - @tagName(mir_ops[2]), - @tagName(mir_ops[3]), - }); - }, - else => |e| return e, - }; - } - switch (mir_tag[0]) { - .f_ => switch (mir_tag[1]) { - .@"2xm1", - .abs, - .add, - .chs, - .clex, - .com, - .comi, - .cos, - .div, - .divr, - .free, - .mul, - .nop, - .prem, - .rndint, - .scale, - .sin, - .sqrt, - .st, - .sub, - .subr, - .tst, - .ucom, - .ucomi, - .wait, - .xam, - .xch, - => {}, - .init, .save => s.top = 0, - .ld, .ptan, .sincos, .xtract => s.top -%= 1, - .patan, .yl2x => s.top +%= 1, - .rstor => unreachable, - else => unreachable, - }, - .f_1 => switch (mir_tag[1]) { - .ld => s.top -%= 1, - .prem => {}, - else => unreachable, - }, - .f_b, .f_be, .f_e, .f_nb, .f_nbe, .f_ne, .f_nu, .f_u => switch (mir_tag[1]) { - .cmov => {}, - else => unreachable, - }, - .f_cw, .f_env, .f_sw => switch (mir_tag[1]) { - .ld, .st => {}, - else => unreachable, - }, - .f_p1 => switch (mir_tag[1]) { - .yl2x => s.top +%= 1, - else => unreachable, - }, - .f_cstp => switch (mir_tag[1]) { - .de => s.top -%= 1, - .in => s.top +%= 1, - else => unreachable, - }, - .f_l2e, .f_l2t, .f_lg2, .f_ln2, .f_pi, .f_z => switch (mir_tag[1]) { - .ld => s.top -%= 1, - else => unreachable, - }, - .f_p => switch (mir_tag[1]) { - .add, .com, .comi, .div, .divr, .mul, .st, .sub, .subr, .ucom, .ucomi => s.top +%= 1, - else => { - const fixes = @tagName(mir_tag[0]); - const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; - std.debug.panic("{s}: {s}{s}{s}\n", .{ - @src().fn_name, - fixes[0..fixes_blank], - @tagName(mir_tag[1]), - fixes[fixes_blank + 1 ..], - }); - }, - }, - .f_pp => switch (mir_tag[1]) { - .com, .ucom => s.top +%= 2, - else => unreachable, - }, - .fb_ => switch (mir_tag[1]) { - .ld => s.top -%= 1, - else => unreachable, - }, - .fb_p => switch (mir_tag[1]) { - .st => s.top +%= 1, - else => unreachable, - }, - .fi_ => switch (mir_tag[1]) { - .add, .com, .div, .divr, .mul, .st, .stt, .sub, .subr => {}, - .ld => s.top -%= 1, - else => unreachable, - }, - .fi_p => switch (mir_tag[1]) { - .com, .st, .stt => s.top +%= 1, - else => unreachable, - }, - .fn_ => switch (mir_tag[1]) { - .clex => {}, - .init, .save => s.top = 0, - else => unreachable, - }, - .fn_cw, .fn_env, .fn_sw => switch (mir_tag[1]) { - .st => {}, - else => unreachable, - }, - .fx_ => switch (mir_tag[1]) { - .rstor => unreachable, - .save => {}, - else => unreachable, - }, - else => {}, - } - } - - fn lowerReg(s: *const Select, reg: Register) Register { - return if (reg.isClass(.x87)) @enumFromInt(@intFromEnum(Register.st0) + (@as(u3, @intCast(reg.enc())) -% s.top)) else reg; - } - - const Case = struct { - required_abi: enum { any, gnu, msvc } = .any, - required_cc_abi: enum { any, sysv64, win64 } = .any, - required_features: [4]?std.Target.x86.Feature = @splat(null), - src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any), - dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any), - patterns: []const Select.Pattern, - call_frame: packed struct(u16) { size: u10 = 0, alignment: InternPool.Alignment } = .{ .size = 0, .alignment = .none }, - extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused), - dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused), - clobbers: packed struct { - eflags: bool = false, - caller_preserved: CallConv = .none, - } = .{}, - each: union(enum) { - once: []const Instruction, - }, - - const CallConv = enum(u2) { none, ccc, zigcc }; - }; - - const Constraint = union(enum) { - any, - any_bool_vec, - any_int, - any_signed_int, - any_unsigned_int, - any_scalar_int, - any_scalar_signed_int, - any_scalar_unsigned_int, - any_float, - po2_any, - bool, - bool_vec: Memory.Size, - exact_bool_vec: u16, - ptr_any_bool_vec, - ptr_bool_vec: Memory.Size, - ptr_any_bool_vec_elem, - ptr_bool_vec_elem: Memory.Size, - remainder_bool_vec: OfIsSizes, - exact_remainder_bool_vec: struct { of: Memory.Size, is: u16 }, - signed_int_vec: Memory.Size, - signed_int_or_full_vec: Memory.Size, - unsigned_int_vec: Memory.Size, - size: Memory.Size, - multiple_size: Memory.Size, - int: Memory.Size, - int_or_float: Memory.Size, - scalar_int_is: Memory.Size, - scalar_signed_int_is: Memory.Size, - scalar_unsigned_int_is: Memory.Size, - scalar: OfIsSizes, - scalar_int: OfIsSizes, - scalar_signed_int: OfIsSizes, - scalar_unsigned_int: OfIsSizes, - exclusive_scalar_int: OfIsSizes, - exclusive_scalar_signed_int: OfIsSizes, - exclusive_scalar_unsigned_int: OfIsSizes, - exact_scalar_int: OfIsSizes, - exact_scalar_signed_int: OfIsSizes, - exact_scalar_unsigned_int: OfIsSizes, - scalar_signed_or_exclusive_int: OfIsSizes, - scalar_exact_int: struct { of: Memory.Size, is: u16 }, - scalar_exact_signed_int: struct { of: Memory.Size, is: u16 }, - scalar_exact_unsigned_int: struct { of: Memory.Size, is: u16 }, - multiple_scalar: OfIsSizes, - multiple_scalar_int: OfIsSizes, - unaligned_multiple_scalar_int: OfIsSizes, - unaligned_multiple_scalar_signed_int: OfIsSizes, - unaligned_multiple_scalar_unsigned_int: OfIsSizes, - multiple_scalar_signed_int: OfIsSizes, - multiple_scalar_unsigned_int: OfIsSizes, - multiple_scalar_signed_or_exclusive_int: OfIsSizes, - multiple_scalar_exact_int: struct { of: Memory.Size, is: u16 }, - multiple_scalar_exact_signed_int: struct { of: Memory.Size, is: u16 }, - multiple_scalar_exact_unsigned_int: struct { of: Memory.Size, is: u16 }, - scalar_remainder_int: OfIsSizes, - scalar_remainder_signed_int: OfIsSizes, - scalar_remainder_unsigned_int: OfIsSizes, - scalar_exact_remainder_int: OfIsSizes, - scalar_exact_remainder_signed_int: OfIsSizes, - scalar_exact_remainder_unsigned_int: OfIsSizes, - float: Memory.Size, - scalar_any_float: Memory.Size, - scalar_float: OfIsSizes, - exclusive_scalar_float: OfIsSizes, - exact_scalar_float: OfIsSizes, - multiple_scalar_any_float: Memory.Size, - multiple_scalar_float: OfIsSizes, - unaligned_multiple_scalar_float: OfIsSizes, - exact_int: u16, - exact_signed_int: u16, - exact_unsigned_int: u16, - signed_or_exact_int: Memory.Size, - unsigned_or_exact_int: Memory.Size, - signed_or_exclusive_int: Memory.Size, - po2_int: Memory.Size, - signed_po2_int: Memory.Size, - unsigned_po2_or_exact_int: Memory.Size, - remainder_int: OfIsSizes, - remainder_signed_int: OfIsSizes, - remainder_unsigned_int: OfIsSizes, - exact_remainder_int: struct { of: Memory.Size, is: u16 }, - exact_remainder_signed_int: struct { of: Memory.Size, is: u16 }, - exact_remainder_unsigned_int: struct { of: Memory.Size, is: u16 }, - signed_or_exact_remainder_int: OfIsSizes, - unsigned_or_exact_remainder_int: OfIsSizes, - signed_int: Memory.Size, - unsigned_int: Memory.Size, - elem_size_is: u8, - po2_elem_size, - elem_int: Memory.Size, - - const OfIsSizes = struct { of: Memory.Size, is: Memory.Size }; - - fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool { - const zcu = cg.pt.zcu; - return switch (constraint) { - .any => true, - .any_bool_vec => ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type, - .any_int => cg.intInfo(ty) != null, - .any_signed_int => if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed else false, - .any_unsigned_int => if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned else false, - .any_scalar_int => cg.intInfo(ty.scalarType(zcu)) != null, - .any_scalar_signed_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed else false, - .any_scalar_unsigned_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned else false, - .any_float => ty.isRuntimeFloat(), - .po2_any => std.math.isPowerOfTwo(ty.abiSize(zcu)), - .bool => ty.toIntern() == .bool_type, - .bool_vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and - size.bitSize(cg.target) >= ty.vectorLen(zcu), - .exact_bool_vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and - size == ty.vectorLen(zcu), - .ptr_any_bool_vec => switch (zcu.intern_pool.indexToKey(ty.childType(zcu).toIntern())) { - .vector_type => |vector_type| vector_type.child == .bool_type, - else => false, - }, - .ptr_bool_vec => |size| switch (zcu.intern_pool.indexToKey(ty.childType(zcu).toIntern())) { - .vector_type => |vector_type| vector_type.child == .bool_type and size.bitSize(cg.target) >= vector_type.len, - else => false, - }, - .ptr_any_bool_vec_elem => { - const ptr_info = ty.ptrInfo(zcu); - return switch (ptr_info.flags.vector_index) { - .none => false, - .runtime => unreachable, - else => ptr_info.child == .bool_type, - }; - }, - .ptr_bool_vec_elem => |size| { - const ptr_info = ty.ptrInfo(zcu); - return switch (ptr_info.flags.vector_index) { - .none => false, - .runtime => unreachable, - else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size, - }; - }, - .remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and - of_is.is.bitSize(cg.target) >= (ty.vectorLen(zcu) - 1) % of_is.of.bitSize(cg.target) + 1, - .exact_remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and - of_is.is == (ty.vectorLen(zcu) - 1) % of_is.of.bitSize(cg.target) + 1, - .signed_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and - if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .signed else false, - .signed_int_or_full_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and - if (cg.intInfo(ty.childType(zcu))) |int_info| switch (int_info.signedness) { - .signed => true, - .unsigned => int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits), - } else false, - .unsigned_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and - if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .unsigned else false, - .size => |size| @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu), - .multiple_size => |size| ty.abiSize(zcu) % @divExact(size.bitSize(cg.target), 8) == 0, - .int => |size| if (cg.intInfo(ty)) |int_info| size.bitSize(cg.target) >= int_info.bits else false, - .int_or_float => |size| if (cg.intInfo(ty)) |int_info| - size.bitSize(cg.target) >= int_info.bits - else if (cg.floatBits(ty)) |float_bits| - size.bitSize(cg.target) == float_bits - else - false, - .scalar_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| - size.bitSize(cg.target) >= int_info.bits - else - false, - .scalar_signed_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { - .signed => size.bitSize(cg.target) >= int_info.bits, - .unsigned => false, - } else false, - .scalar_unsigned_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { - .signed => false, - .unsigned => size.bitSize(cg.target) >= int_info.bits, - } else false, - .scalar => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| - of_is.is.bitSize(cg.target) >= int_info.bits - else if (cg.floatBits(ty.scalarType(zcu))) |float_bits| - of_is.is.bitSize(cg.target) == float_bits - else - false, - .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, - .scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .exclusive_scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, - .exclusive_scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .exclusive_scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .exact_scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, - .exact_scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .exact_scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .scalar_signed_or_exclusive_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => of_is.is.bitSize(cg.target) >= int_info.bits, - .unsigned => of_is.is.bitSize(cg.target) > int_info.bits, - } else false, - .scalar_exact_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is == int_info.bits else false, - .scalar_exact_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is == int_info.bits else false, - .scalar_exact_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is == int_info.bits else false, - .multiple_scalar => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| - of_is.is.bitSize(cg.target) >= int_info.bits - else if (cg.floatBits(ty.scalarType(zcu))) |float_bits| - of_is.is.bitSize(cg.target) == float_bits - else - false, - .multiple_scalar_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, - .unaligned_multiple_scalar_int => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, - .unaligned_multiple_scalar_signed_int => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .unaligned_multiple_scalar_unsigned_int => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .multiple_scalar_signed_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .multiple_scalar_unsigned_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= int_info.bits else false, - .multiple_scalar_signed_or_exclusive_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { - .signed => of_is.is.bitSize(cg.target) >= int_info.bits, - .unsigned => of_is.is.bitSize(cg.target) > int_info.bits, - } else false, - .multiple_scalar_exact_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is == int_info.bits else false, - .multiple_scalar_exact_signed_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is == int_info.bits else false, - .multiple_scalar_exact_unsigned_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is == int_info.bits else false, - .scalar_remainder_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| - of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 - else - false, - .scalar_remainder_signed_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .scalar_remainder_unsigned_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .scalar_exact_remainder_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| - of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 - else - false, - .scalar_exact_remainder_signed_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .scalar_exact_remainder_unsigned_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .float => |size| if (cg.floatBits(ty)) |float_bits| size.bitSize(cg.target) == float_bits else false, - .scalar_any_float => |size| @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and - cg.floatBits(ty.scalarType(zcu)) != null, - .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, - .exclusive_scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and - if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, - .exact_scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and - if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, - .multiple_scalar_any_float => |size| ty.abiSize(zcu) % @divExact(size.bitSize(cg.target), 8) == 0 and - cg.floatBits(ty.scalarType(zcu)) != null, - .multiple_scalar_float => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, - .unaligned_multiple_scalar_float => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and - if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, - .exact_int => |bit_size| if (cg.intInfo(ty)) |int_info| bit_size == int_info.bits else false, - .exact_signed_int => |bit_size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => bit_size == int_info.bits, - .unsigned => false, - } else false, - .exact_unsigned_int => |bit_size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => false, - .unsigned => bit_size == int_info.bits, - } else false, - .signed_or_exact_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => size.bitSize(cg.target) >= int_info.bits, - .unsigned => size.bitSize(cg.target) == int_info.bits, - } else false, - .unsigned_or_exact_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => size.bitSize(cg.target) == int_info.bits, - .unsigned => size.bitSize(cg.target) >= int_info.bits, - } else false, - .signed_or_exclusive_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => size.bitSize(cg.target) >= int_info.bits, - .unsigned => size.bitSize(cg.target) > int_info.bits, - } else false, - .po2_int => |size| if (cg.intInfo(ty)) |int_info| - std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits - else - false, - .signed_po2_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits, - .unsigned => false, - } else false, - .unsigned_po2_or_exact_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => size.bitSize(cg.target) == int_info.bits, - .unsigned => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits, - } else false, - .remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| - of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 - else - false, - .remainder_signed_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed and - of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .remainder_unsigned_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned and - of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| - of_is.is == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 - else - false, - .exact_remainder_signed_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed and - of_is.is == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .exact_remainder_unsigned_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned and - of_is.is == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, - .signed_or_exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, - .unsigned => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, - } else false, - .unsigned_or_exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, - .unsigned => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, - } else false, - .signed_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => size.bitSize(cg.target) >= int_info.bits, - .unsigned => false, - } else false, - .unsigned_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { - .signed => false, - .unsigned => size.bitSize(cg.target) >= int_info.bits, - } else false, - .elem_size_is => |size| size == ty.elemType2(zcu).abiSize(zcu), - .po2_elem_size => std.math.isPowerOfTwo(ty.elemType2(zcu).abiSize(zcu)), - .elem_int => |size| if (cg.intInfo(ty.elemType2(zcu))) |elem_int_info| - size.bitSize(cg.target) >= elem_int_info.bits - else - false, - }; - } - }; - - const Pattern = struct { - src: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Src, - commute: struct { u8, u8 } = .{ 0, 0 }, - - const Src = union(enum) { - none, - any, - imm: i16, - imm8, - imm16, - imm32, - simm32, - to_reg: Register, - to_reg_pair: [2]Register, - to_param_gpr: TempSpec.Kind.CallConvRegSpec, - to_param_gpr_pair: TempSpec.Kind.CallConvRegSpec, - to_param_sse: TempSpec.Kind.CallConvRegSpec, - to_ret_gpr: TempSpec.Kind.CallConvRegSpec, - to_ret_gpr_pair: TempSpec.Kind.CallConvRegSpec, - mem, - to_mem, - mut_mem, - to_mut_mem, - gpr, - immut_gpr, - to_gpr, - mut_gpr, - to_mut_gpr, - to_gphi, - to_mut_gphi, - x87, - to_x87, - mut_x87, - to_mut_x87, - mmx, - to_mmx, - mut_mmx, - to_mut_mmx, - mm, - to_mm, - mut_mm, - to_mut_mm, - sse, - to_sse, - mut_sse, - to_mut_sse, - xmm, - to_xmm, - mut_xmm, - to_mut_xmm, - ymm, - to_ymm, - mut_ymm, - to_mut_ymm, - reg_mask: RegMaskSpec, - all_reg_mask: RegMaskSpec, - - const RegMaskSpec = struct { - size: Memory.Size, - is: enum { - any, - uninverted, - inverted, - - fn matches(is: @This(), inverted: bool) bool { - return switch (is) { - .any => true, - .uninverted => !inverted, - .inverted => inverted, - }; - } - } = .any, - }; - - fn matches(src: Src, temp: Temp, cg: *CodeGen) bool { - return switch (src) { - .none => temp.tracking(cg).short == .none, - .any => true, - .imm => |specific_imm| switch (temp.tracking(cg).short) { - .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm, - else => false, - }, - .imm8 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(u8, imm) != null or - std.math.cast(i8, @as(i64, @bitCast(imm))) != null, - else => false, - }, - .imm16 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(u16, imm) != null or - std.math.cast(i16, @as(i64, @bitCast(imm))) != null, - else => false, - }, - .imm32 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(u32, imm) != null or - std.math.cast(i32, @as(i64, @bitCast(imm))) != null, - else => false, - }, - .simm32 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(i32, @as(i64, @bitCast(imm))) != null, - else => false, - }, - .mem => temp.tracking(cg).short.isMemory(), - .to_mem, .to_mut_mem => true, - .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(), - .to_reg, .to_reg_pair, .to_param_gpr, .to_param_gpr_pair, .to_param_sse, .to_ret_gpr, .to_ret_gpr_pair => true, - .gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.general_purpose), - .register_offset => |reg_off| reg_off.reg.isClass(.general_purpose) and reg_off.off == 0, - else => false, - }, - .immut_gpr => !temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.general_purpose), - .register_offset => |reg_off| reg_off.reg.isClass(.general_purpose) and reg_off.off == 0, - else => false, - }, - .mut_gpr => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.general_purpose), - .register_offset => |reg_off| reg_off.reg.isClass(.general_purpose) and reg_off.off == 0, - else => false, - }, - .to_gpr, .to_mut_gpr, .to_gphi, .to_mut_gphi => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8, - .x87 => switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.x87), - .register_offset => |reg_off| reg_off.reg.isClass(.x87) and reg_off.off == 0, - else => false, - }, - .mut_x87 => temp.isMut(cg) and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.x87), - .register_offset => |reg_off| reg_off.reg.isClass(.x87) and reg_off.off == 0, - else => false, - }, - .to_x87, .to_mut_x87 => true, - .mmx => switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.mmx), - .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, - else => false, - }, - .mut_mmx => temp.isMut(cg) and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.mmx), - .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, - else => false, - }, - .to_mmx, .to_mut_mmx => true, - .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.mmx), - .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, - else => false, - }, - .mut_mm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.mmx), - .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, - else => false, - }, - .to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8, - .sse => switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.sse), - .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, - else => false, - }, - .mut_sse => temp.isMut(cg) and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.sse), - .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, - else => false, - }, - .to_sse, .to_mut_sse => true, - .xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.sse), - .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, - else => false, - }, - .mut_xmm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.sse), - .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, - else => false, - }, - .to_xmm, .to_mut_xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16, - .ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.sse), - .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, - else => false, - }, - .mut_ymm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) { - .register => |reg| reg.isClass(.sse), - .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, - else => false, - }, - .to_ymm, .to_mut_ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32, - .reg_mask => |mask_spec| switch (temp.tracking(cg).short) { - .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) >= - reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and - mask_spec.is.matches(reg_mask.info.inverted), - else => false, - }, - .all_reg_mask => |mask_spec| switch (temp.tracking(cg).short) { - .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) == - reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and - reg_mask.info.kind == .all and mask_spec.is.matches(reg_mask.info.inverted), - else => false, - }, - }; - } - - fn convert(src: Src, temp: *Temp, cg: *CodeGen) InnerError!bool { - return switch (src) { - .none, .any, .imm, .imm8, .imm16, .imm32, .simm32, .reg_mask, .all_reg_mask => false, - .mem, .to_mem => try temp.toBase(false, cg), - .mut_mem, .to_mut_mem => try temp.toBase(true, cg), - .to_reg => |reg| try temp.toReg(reg, cg), - .to_reg_pair => |regs| try temp.toRegPair(regs, cg), - .to_param_gpr => |param_spec| try temp.toReg(abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)], cg), - .to_param_gpr_pair => |param_spec| try temp.toRegPair(abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..2].*, cg), - .to_param_sse => |param_spec| try temp.toReg(abi.getCAbiSseParamRegs(param_spec.tag(cg), cg.target)[param_spec.index(cg)], cg), - .to_ret_gpr => |ret_spec| try temp.toReg(abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)], cg), - .to_ret_gpr_pair => |ret_spec| try temp.toRegPair(abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..2].*, cg), - .gpr, .immut_gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg), - .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg), - .to_gphi => try temp.toRegClass(false, .gphi, cg), - .to_mut_gphi => try temp.toRegClass(true, .gphi, cg), - .x87, .to_x87 => try temp.toRegClass(false, .x87, cg), - .mut_x87, .to_mut_x87 => try temp.toRegClass(true, .x87, cg), - .mmx, .to_mmx, .mm, .to_mm => try temp.toRegClass(false, .mmx, cg), - .mut_mmx, .to_mut_mmx, .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg), - .sse, .to_sse, .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg), - .mut_sse, .to_mut_sse, .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg), - }; - } - }; - }; - - const TempSpec = struct { - type: Type = .noreturn, - kind: Kind, - - const unused: TempSpec = .{ .kind = .unused }; - - const Kind = union(enum) { - unused, - any, - none, - undef, - cc: Condition, - ptest_cc: struct { ref: Select.Operand.Ref, not: bool = false, all: u1 }, - imm: i32, - ref: Select.Operand.Ref, - reg: Register, - reg_pair: [2]Register, - param_gpr: CallConvRegSpec, - param_gpr_pair: CallConvRegSpec, - param_sse: CallConvRegSpec, - ret_gpr: CallConvRegSpec, - ret_gpr_pair: CallConvRegSpec, - ret_sse: CallConvRegSpec, - rc: Register.Class, - rc_pair: Register.Class, - mut_rc: struct { ref: Select.Operand.Ref, rc: Register.Class }, - ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, - rc_mask: struct { rc: Register.Class, info: MaskInfo }, - mut_rc_mask: struct { ref: Select.Operand.Ref, rc: Register.Class, info: MaskInfo }, - mem, - mem_of_type: Select.Operand.Ref, - smin_mem: ConstSpec, - smax_mem: ConstSpec, - slimit_delta_mem: ConstSpec, - umin_mem: ConstSpec, - umax_mem: ConstSpec, - umax_delta_mem: ConstSpec, - @"0x1p63_mem": ConstSpec, - f64_0x1p52_0x1p84_mem, - u32_0x1p52_hi_0x1p84_hi_0_0_mem, - f32_0_0x1p64_mem, - pshufb_splat_mem: struct { of: Memory.Size = .none, size: Memory.Size }, - pshufb_trunc_mem: struct { of: Memory.Size = .none, from: Memory.Size, to: Memory.Size }, - pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size }, - pand_mask_mem: struct { ref: Select.Operand.Ref, invert: bool = false }, - ptest_mask_mem: Select.Operand.Ref, - pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 }, - bits_mem: enum { forward, reverse }, - splat_int_mem: struct { ref: Select.Operand.Ref, inside: enum { umin, smin, smax } = .umin, outside: enum { smin, smax } }, - splat_float_mem: struct { ref: Select.Operand.Ref, inside: enum { zero } = .zero, outside: f16 }, - frame: FrameIndex, - lazy_sym: struct { kind: link.File.LazySymbol.Kind, ref: Select.Operand.Ref = .none }, - extern_func: [*:0]const u8, - - const ConstSpec = struct { - ref: Select.Operand.Ref = .none, - to_signedness: ?std.builtin.Signedness = null, - vectorize_to: ?Memory.Size = null, - }; - - const CallConvRegSpec = struct { - cc: Case.CallConv, - after: u2, - at: u2, - - fn tag(spec: CallConvRegSpec, cg: *const CodeGen) std.builtin.CallingConvention.Tag { - return switch (spec.cc) { - .none => unreachable, - .ccc => cg.target.cCallingConvention().?, - .zigcc => .auto, - }; - } - - fn index(spec: CallConvRegSpec, cg: *const CodeGen) u2 { - return switch (spec.tag(cg)) { - .auto, .x86_64_sysv => spec.after, - .x86_64_win => spec.at, - else => unreachable, - }; - } - }; - - fn lock(kind: Kind, cg: *CodeGen) ![2]?RegisterLock { - var reg_locks: [2]?RegisterLock = @splat(null); - const regs: [2]Register = switch (kind) { - else => return reg_locks, - .reg => |reg| .{ reg, .none }, - .reg_pair => |regs| regs, - .param_gpr => |param_spec| abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..1].* ++ .{.none}, - .param_gpr_pair => |param_spec| abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..2].*, - .param_sse => |param_spec| abi.getCAbiSseParamRegs(param_spec.tag(cg), cg.target)[param_spec.index(cg)..][0..1].* ++ .{.none}, - .ret_gpr => |ret_spec| abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..1].* ++ .{.none}, - .ret_gpr_pair => |ret_spec| abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..2].*, - .ret_sse => |ret_spec| abi.getCAbiSseReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..1].* ++ .{.none}, - }; - for (regs, ®_locks) |reg, *reg_lock| { - if (reg == .none) continue; - const reg_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue; - try cg.register_manager.getRegIndex(reg_index, null); - reg_lock.* = cg.register_manager.lockRegIndex(reg_index); - } - return reg_locks; - } - }; - - fn create(spec: TempSpec, s: *const Select) InnerError!struct { Temp, bool } { - const cg = s.cg; - const pt = cg.pt; - return switch (spec.kind) { - .unused => .{ undefined, false }, - .any => .{ try cg.tempAlloc(spec.type), true }, - .none => .{ try cg.tempInit(spec.type, .none), true }, - .undef => .{ try cg.tempInit(spec.type, .undef), true }, - .imm => |imm| .{ try cg.tempInit(spec.type, .{ .immediate = @bitCast(@as(i64, imm)) }), true }, - .cc => |cc| .{ try cg.tempInit(spec.type, .{ .eflags = cc }), true }, - .ptest_cc => |cc_spec| { - const ccs: [2]Condition = .{ .z, .c }; - const cc = ccs[cc_spec.all ^ @intFromBool(cc_spec.ref.valueOf(s).register_mask.info.inverted)]; - return .{ try cg.tempInit(spec.type, .{ .eflags = switch (cc_spec.not) { - false => cc, - true => cc.negate(), - } }), true }; - }, - .ref => |ref| .{ ref.tempOf(s), false }, - .reg => |reg| .{ try cg.tempInit(spec.type, .{ .register = reg }), true }, - .reg_pair => |regs| .{ try cg.tempInit(spec.type, .{ .register_pair = regs }), true }, - .param_gpr => |param_spec| .{ try cg.tempInit(spec.type, .{ - .register = abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)], - }), true }, - .param_gpr_pair => |param_spec| .{ try cg.tempInit(spec.type, .{ - .register_pair = abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..2].*, - }), true }, - .param_sse => |param_spec| .{ try cg.tempInit(spec.type, .{ - .register = abi.getCAbiSseParamRegs(param_spec.tag(cg), cg.target)[param_spec.index(cg)], - }), true }, - .ret_gpr => |ret_spec| .{ try cg.tempInit(spec.type, .{ - .register = abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)], - }), true }, - .ret_gpr_pair => |ret_spec| .{ try cg.tempInit(spec.type, .{ - .register_pair = abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..2].*, - }), true }, - .ret_sse => |ret_spec| .{ try cg.tempInit(spec.type, .{ - .register = abi.getCAbiSseReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)], - }), true }, - .rc => |rc| .{ try cg.tempAllocReg(spec.type, regSetForRegClass(rc)), true }, - .rc_pair => |rc| .{ try cg.tempAllocRegPair(spec.type, regSetForRegClass(rc)), true }, - .mut_rc => |ref_rc| { - const temp = ref_rc.ref.tempOf(s); - if (temp.isMut(cg)) switch (temp.tracking(cg).short) { - .register => |reg| if (reg.isClass(ref_rc.rc)) return .{ temp, false }, - .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.isClass(ref_rc.rc)) return .{ temp, false }, - else => {}, - }; - return .{ try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc)), true }; - }, - .ref_mask => |ref_mask| .{ ref_mask.ref.tempOf(s), false }, - .rc_mask => |rc_mask| .{ try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)), true }, - .mut_rc_mask => |ref_rc_mask| { - const temp = ref_rc_mask.ref.tempOf(s); - if (temp.isMut(cg)) switch (temp.tracking(cg).short) { - .register => |reg| if (reg.isClass(ref_rc_mask.rc)) return .{ temp, false }, - .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.isClass(ref_rc_mask.rc)) return .{ temp, false }, - else => {}, - }; - return .{ try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc_mask.rc)), true }; - }, - .mem => .{ try cg.tempAllocMem(spec.type), true }, - .mem_of_type => |ref| .{ try cg.tempAllocMem(ref.typeOf(s)), true }, - .smin_mem, - .smax_mem, - .slimit_delta_mem, - .umin_mem, - .umax_mem, - .umax_delta_mem, - .@"0x1p63_mem", - => |const_spec| { - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty = if (const_spec.ref == .none) spec.type else const_spec.ref.typeOf(s); - const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) { - else => .{ null, ty }, - .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) }, - }; - const res_vector_len: ?u32 = if (const_spec.vectorize_to) |vectorize_to| switch (vectorize_to) { - .none => null, - else => @intCast(@divExact(@divExact(vectorize_to.bitSize(cg.target), 8), scalar_ty.abiSize(pt.zcu))), - } else vector_len; - const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) { - .bool_type => .{ scalar_ty, .fromInterned(switch (spec.kind) { - else => unreachable, - .smin_mem, .umax_mem => .bool_true, - .smax_mem, .umin_mem => .bool_false, - }) }, - else => { - const scalar_info: std.builtin.Type.Int = cg.intInfo(scalar_ty) orelse .{ - .signedness = .signed, - .bits = cg.floatBits(scalar_ty).?, - }; - const res_scalar_info: std.builtin.Type.Int = .{ - .signedness = const_spec.to_signedness orelse scalar_info.signedness, - .bits = switch (spec.kind) { - else => scalar_info.bits, - .slimit_delta_mem, .umax_delta_mem => cg.intInfo(spec.type.scalarType(zcu)).?.bits, - }, - }; - const res_scalar_ty = try pt.intType(res_scalar_info.signedness, res_scalar_info.bits); - - if (res_scalar_info.bits <= 64) { - const int_val: i64 = switch (spec.kind) { - else => unreachable, - .smin_mem => std.math.minInt(i64), - .smax_mem => std.math.maxInt(i64), - .slimit_delta_mem => std.math.maxInt(i64) - - (@as(i64, std.math.maxInt(i64)) >> @intCast(res_scalar_info.bits - scalar_info.bits)), - .umin_mem => 0, - .umax_mem => -1, - .umax_delta_mem => @bitCast(std.math.maxInt(u64) - - (@as(u64, std.math.maxInt(u64)) >> @intCast(res_scalar_info.bits - scalar_info.bits))), - .@"0x1p63_mem" => switch (res_scalar_info.bits) { - else => unreachable, - 16 => @as(i64, @as(i16, @bitCast(@as(f16, 0x1p63)))) << 64 - 16, - 32 => @as(i64, @as(i32, @bitCast(@as(f32, 0x1p63)))) << 64 - 32, - 64 => @as(i64, @as(i64, @bitCast(@as(f64, 0x1p63)))) << 64 - 64, - }, - }; - const shift: u6 = @intCast(64 - res_scalar_info.bits); - break :res_scalar .{ res_scalar_ty, switch (res_scalar_info.signedness) { - .signed => try pt.intValue_i64(res_scalar_ty, int_val >> shift), - .unsigned => try pt.intValue_u64(res_scalar_ty, @as(u64, @bitCast(int_val)) >> shift), - } }; - } - - const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(1 << 10)]std.math.big.Limb; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); - const allocator = stack.get(); - var res_big_int: std.math.big.int.Mutable = .{ - .limbs = try allocator.alloc( - std.math.big.Limb, - std.math.big.int.calcTwosCompLimbCount(res_scalar_info.bits), - ), - .len = undefined, - .positive = undefined, - }; - defer allocator.free(res_big_int.limbs); - switch (spec.kind) { - else => unreachable, - .smin_mem, .smax_mem, .umin_mem, .umax_mem => res_big_int.setTwosCompIntLimit(switch (spec.kind) { - else => unreachable, - .smin_mem, .umin_mem => .min, - .smax_mem, .umax_mem => .max, - }, switch (spec.kind) { - else => unreachable, - .smin_mem, .smax_mem => .signed, - .umin_mem, .umax_mem => .unsigned, - }, res_scalar_info.bits), - .slimit_delta_mem, .umax_delta_mem => { - var big_int: std.math.big.int.Mutable = .{ - .limbs = try allocator.alloc( - std.math.big.Limb, - std.math.big.int.calcTwosCompLimbCount(scalar_info.bits), - ), - .len = undefined, - .positive = undefined, - }; - defer allocator.free(big_int.limbs); - const signedness: std.builtin.Signedness = switch (spec.kind) { - else => unreachable, - .slimit_delta_mem => .signed, - .umax_delta_mem => .unsigned, - }; - big_int.setTwosCompIntLimit(.max, signedness, scalar_info.bits); - res_big_int.setTwosCompIntLimit(.max, signedness, res_scalar_info.bits); - res_big_int.sub(res_big_int.toConst(), big_int.toConst()); - }, - .@"0x1p63_mem" => switch (res_scalar_info.bits) { - else => unreachable, - 80 => res_big_int.set(@as(u80, @bitCast(@as(f80, 0x1p63)))), - 128 => res_big_int.set(@as(u128, @bitCast(@as(f128, 0x1p63)))), - }, - } - res_big_int.truncate(res_big_int.toConst(), res_scalar_info.signedness, res_scalar_info.bits); - break :res_scalar .{ res_scalar_ty, try pt.intValue_big(res_scalar_ty, res_big_int.toConst()) }; - }, - }; - const res_val = if (res_vector_len) |len| try pt.aggregateSplatValue(try pt.vectorType(.{ - .len = len, - .child = res_scalar_ty.toIntern(), - }), res_scalar_val) else res_scalar_val; - return .{ try cg.tempMemFromValue(res_val), true }; - }, - .f64_0x1p52_0x1p84_mem => .{ try cg.tempMemFromValue( - try pt.aggregateValue(try pt.vectorType(.{ .len = 2, .child = .f64_type }), &.{ - (try pt.floatValue(.f64, @as(f64, 0x1p52))).toIntern(), - (try pt.floatValue(.f64, @as(f64, 0x1p84))).toIntern(), - }), - ), true }, - .u32_0x1p52_hi_0x1p84_hi_0_0_mem => .{ try cg.tempMemFromValue( - try pt.aggregateValue(try pt.vectorType(.{ .len = 4, .child = .u32_type }), &(.{ - (try pt.intValue(.u32, @as(u64, @bitCast(@as(f64, 0x1p52))) >> 32)).toIntern(), - (try pt.intValue(.u32, @as(u64, @bitCast(@as(f64, 0x1p84))) >> 32)).toIntern(), - } ++ .{(try pt.intValue(.u32, 0)).toIntern()} ** 2)), - ), true }, - .f32_0_0x1p64_mem => .{ try cg.tempMemFromValue( - try pt.aggregateValue(try pt.vectorType(.{ .len = 2, .child = .f32_type }), &.{ - (try pt.floatValue(.f32, @as(f32, 0))).toIntern(), - (try pt.floatValue(.f32, @as(f32, 0x1p64))).toIntern(), - }), - ), true }, - .pshufb_splat_mem => |splat_spec| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - var elem_buf: [32]u8 = @splat(1 << 7); - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const of_bytes: u6 = @intCast(switch (splat_spec.of) { - .none => elems.len, - else => |of| @divExact(of.bitSize(cg.target), 8), - }); - const size_bytes: u6 = @intCast(@divExact(splat_spec.size.bitSize(cg.target), 8)); - var index: u6 = 0; - while (index < of_bytes) : (index += size_bytes) for (0..size_bytes) |byte_off| { - elems[index + byte_off] = @as(u4, @truncate(byte_off)); - }; - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .pshufb_trunc_mem => |trunc_spec| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - var elem_buf: [32]u8 = @splat(1 << 7); - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const of_bytes: u6 = @intCast(switch (trunc_spec.of) { - .none => elems.len, - else => |of| @divExact(of.bitSize(cg.target), 8), - }); - const from_bytes: u5 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8)); - const to_bytes: u5 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8)); - var from_index: u6 = 0; - var to_index: u6 = 0; - while (from_index < of_bytes) : ({ - from_index += from_bytes; - switch (from_index % 16) { - 0 => to_index = from_index, - else => to_index += to_bytes, - } - }) for (0..to_bytes) |byte_off| { - elems[to_index + byte_off] = @as(u4, @truncate(from_index + byte_off)); - }; - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .pand_trunc_mem => |trunc_spec| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - var elem_buf: [32]u8 = @splat(0); - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const from_bytes: u6 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8)); - const to_bytes: u6 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8)); - var index: u6 = 0; - while (index < elems.len) : (index += from_bytes) @memset(elems[index..][0..to_bytes], std.math.maxInt(u8)); - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .pand_mask_mem => |mask_spec| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - const ref_ty = mask_spec.ref.typeOf(s); - assert(ref_ty.isVector(zcu)); - var elem_buf: [64]u8 = undefined; - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const mask_len: usize = @intCast((cg.unalignedSize(ref_ty) - 1) % elems.len + 1); - const invert_mask: u8 = switch (mask_spec.invert) { - false => std.math.minInt(u8), - true => std.math.maxInt(u8), - }; - @memset(elems[0..mask_len], ~invert_mask); - @memset(elems[mask_len..], invert_mask); - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .ptest_mask_mem => |mask_ref| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - const ref_ty = mask_ref.typeOf(s); - assert(ref_ty.isVector(zcu) and ref_ty.childType(zcu).toIntern() == .bool_type); - const mask_info = mask_ref.valueOf(s).register_mask.info; - var elem_buf: [64]u8 = @splat(0); - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const elem_bytes: u6 = @intCast(@divExact(mask_info.scalar.bitSize(cg.target), 8)); - var index: u7 = 0; - for (0..@intCast(ref_ty.vectorLen(zcu))) |_| { - switch (mask_info.kind) { - .sign => { - @memset(elems[index..][0 .. elem_bytes - 1], std.math.minInt(u8)); - elems[index + elem_bytes - 1] = @bitCast(@as(i8, std.math.minInt(i8))); - }, - .all => @memset(elems[index..][0..elem_bytes], std.math.maxInt(u8)), - } - index += elem_bytes; - } - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .pshufb_bswap_mem => |bswap_spec| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - var elem_buf: [32]u8 = @splat(1 << 7); - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8)); - var to_index: u6 = 0; - for (0..bswap_spec.repeat) |_| for (0..len) |from_index| { - @memset(elems[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index)); - to_index += bswap_spec.smear; - }; - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .bits_mem => |direction| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); - var bytes: [32]u8 = undefined; - const elems = bytes[0..spec.type.vectorLen(zcu)]; - for (elems, 0..) |*elem, index| elem.* = switch (direction) { - .forward => @as(u8, 1 << 0) << @truncate(index), - .reverse => @as(u8, 1 << 7) >> @truncate(index), - }; - return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ - .ty = spec.type.toIntern(), - .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, - } }))), true }; - }, - .splat_int_mem => |splat_spec| { - const zcu = pt.zcu; - const elem_ty = spec.type.scalarType(zcu); - const elem_bits = cg.intInfo(elem_ty).?.bits; - const ref_ty = splat_spec.ref.typeOf(s); - assert(ref_ty.isVector(zcu) and cg.intInfo(ref_ty.childType(zcu)).?.bits <= elem_bits); - var elem_buf: [64]InternPool.Index = undefined; - const elems = elem_buf[0..if (elem_ty.toIntern() == spec.type.toIntern()) 1 else spec.type.vectorLen(zcu)]; - const inside_len = (ref_ty.vectorLen(zcu) - 1) % elems.len + 1; - @memset(elems[0..inside_len], (try pt.intValue(elem_ty, @as(i64, switch (splat_spec.inside) { - .umin => 0, - .smin => std.math.minInt(i64), - .smax => std.math.maxInt(i64), - }) >> @intCast(64 - elem_bits))).toIntern()); - @memset(elems[inside_len..], (try pt.intValue(elem_ty, @as(i64, switch (splat_spec.outside) { - .smin => std.math.minInt(i64), - .smax => std.math.maxInt(i64), - }) >> @intCast(64 - elem_bits))).toIntern()); - if (elems.len == 1) return .{ try cg.tempMemFromValue(.fromInterned(elems[0])), true }; - const mem_size = cg.unalignedSize(spec.type); - return .{ try cg.tempMemFromAlignedValue( - if (mem_size < 16) .fromByteUnits(mem_size) else .none, - try pt.aggregateValue(if (mem_size < 16) try pt.arrayType(.{ - .len = elems.len, - .child = elem_ty.toIntern(), - }) else spec.type, elems), - ), true }; - }, - .splat_float_mem => |splat_spec| { - const zcu = pt.zcu; - assert(spec.type.isVector(zcu)); - const elem_ty = spec.type.childType(zcu); - assert(cg.floatBits(elem_ty) != null); - const ref_ty = splat_spec.ref.typeOf(s); - assert(ref_ty.isVector(zcu) and ref_ty.childType(zcu).toIntern() == elem_ty.toIntern()); - var elem_buf: [@divExact(64, 2)]InternPool.Index = undefined; - const elems = elem_buf[0..spec.type.vectorLen(zcu)]; - const inside_len = (ref_ty.vectorLen(zcu) - 1) % elems.len + 1; - @memset(elems[0..inside_len], (try pt.floatValue(elem_ty, @as(f16, switch (splat_spec.inside) { - .zero => 0.0, - }))).toIntern()); - @memset(elems[inside_len..], (try pt.floatValue(elem_ty, splat_spec.outside)).toIntern()); - return .{ try cg.tempMemFromValue(try pt.aggregateValue(spec.type, elems)), true }; - }, - .frame => |frame_index| .{ try cg.tempInit(spec.type, .{ .load_frame = .{ - .index = frame_index, - .off = switch (frame_index) { - .call_frame => switch (cg.target.cCallingConvention().?) { - .x86_64_sysv => 0, - .x86_64_win => win64_shadow_space, - else => unreachable, - }, - else => 0, - }, - } }), true }, - .lazy_sym => |lazy_symbol_spec| { - const ip = &pt.zcu.intern_pool; - const ty = if (lazy_symbol_spec.ref == .none) spec.type else lazy_symbol_spec.ref.typeOf(s); - return .{ try cg.tempInit(.usize, .{ .lea_lazy_sym = .{ - .kind = lazy_symbol_spec.kind, - .ty = switch (ip.indexToKey(ty.toIntern())) { - .inferred_error_set_type => |func_index| switch (ip.funcIesResolvedUnordered(func_index)) { - .none => unreachable, - else => |ty_index| ty_index, - }, - else => ty.toIntern(), - }, - } }), true }; - }, - .extern_func => |extern_func_spec| .{ try cg.tempInit(spec.type, .{ .lea_extern_func = try cg.addString(std.mem.span(extern_func_spec)) }), true }, - }; - } - - fn finish(spec: TempSpec, temp: *Temp, cg: *CodeGen) InnerError!void { - switch (spec.kind) { - else => {}, - inline .rc_mask, .mut_rc_mask, .ref_mask => |mask| temp.asMask(mask.info, cg), - } - cg.temp_type[@intFromEnum(temp.unwrap(cg).temp)] = spec.type; - } - }; - - const Instruction = struct { - Label, - Mir.Inst.Fixes, - Mir.Inst.Tag, - Select.Operand, - Select.Operand, - Select.Operand, - Select.Operand, - }; - const Label = enum { @"0:", @"1:", @"2:", @"3:", @"4:", @"_", pseudo }; - const Operand = struct { - flags: packed struct(u32) { - tag: Tag, - adjust: Adjust = .none, - base: Ref.Sized = .none, - index: packed struct(u7) { - ref: Ref, - scale: Memory.Scale = .@"1", - } = .{ .ref = .none }, - unused: u2 = 0, - }, - imm: i32 = 0, - - const Tag = enum(u3) { - none, - label, - ref, - simm, - uimm, - lea, - mem, - }; - const Adjust = packed struct(u11) { - sign: enum(u1) { neg, pos }, - lhs: enum(u6) { - none, - ptr_size, - ptr_bit_size, - size, - src0_size, - dst0_size, - delta_size, - delta_elem_size, - unaligned_size, - unaligned_size_add_elem_size, - unaligned_size_sub_elem_size, - unaligned_size_sub_2_elem_size, - bit_size, - src0_bit_size, - @"8_size_sub_bit_size", - len, - elem_limbs, - elem_size, - src0_elem_size, - dst0_elem_size, - src0_elem_size_mul_src1, - vector_index, - src1, - src1_sub_bit_size, - log2_src0_elem_size, - elem_mask, - smin, - smax, - umax, - smin_shr_src1, - smax_shr_src1, - umax_shr_src1, - repeat, - }, - op: enum(u2) { mul, div, div_8_down, rem_8_mul }, - rhs: Memory.Scale, - - const none: Adjust = .{ .sign = .pos, .lhs = .none, .op = .mul, .rhs = .@"1" }; - const sub_ptr_size: Adjust = .{ .sign = .neg, .lhs = .ptr_size, .op = .mul, .rhs = .@"1" }; - const add_ptr_bit_size: Adjust = .{ .sign = .pos, .lhs = .ptr_bit_size, .op = .mul, .rhs = .@"1" }; - const add_size: Adjust = .{ .sign = .pos, .lhs = .size, .op = .mul, .rhs = .@"1" }; - const add_size_div_4: Adjust = .{ .sign = .pos, .lhs = .size, .op = .div, .rhs = .@"4" }; - const add_size_div_8: Adjust = .{ .sign = .pos, .lhs = .size, .op = .div, .rhs = .@"8" }; - const sub_size_div_8: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"8" }; - const sub_size_div_4: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"4" }; - const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" }; - const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" }; - const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; - const add_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; - const add_8_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"8" }; - const add_dst0_size: Adjust = .{ .sign = .pos, .lhs = .dst0_size, .op = .mul, .rhs = .@"1" }; - const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" }; - const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" }; - const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" }; - const add_unaligned_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" }; - const sub_unaligned_size: Adjust = .{ .sign = .neg, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" }; - const add_unaligned_size_add_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_add_elem_size, .op = .mul, .rhs = .@"1" }; - const add_unaligned_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_elem_size, .op = .mul, .rhs = .@"1" }; - const add_unaligned_size_sub_2_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_2_elem_size, .op = .mul, .rhs = .@"1" }; - const add_2_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"2" }; - const add_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"1" }; - const add_bit_size_rem_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"1" }; - const add_bit_size_rem_64: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"8" }; - const add_bit_size_div_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div, .rhs = .@"8" }; - const add_bit_size_div_8_down_1: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div_8_down, .rhs = .@"1" }; - const add_bit_size_div_8_down_2: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div_8_down, .rhs = .@"2" }; - const add_bit_size_div_8_down_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div_8_down, .rhs = .@"8" }; - const sub_bit_size_rem_64: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"8" }; - const sub_bit_size: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .mul, .rhs = .@"1" }; - const add_src0_bit_size: Adjust = .{ .sign = .pos, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" }; - const sub_src0_bit_size: Adjust = .{ .sign = .neg, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" }; - const add_bit_size_sub_8_size: Adjust = .{ .sign = .neg, .lhs = .@"8_size_sub_bit_size", .op = .mul, .rhs = .@"1" }; - const add_8_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"8" }; - const add_4_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"4" }; - const add_3_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"3" }; - const add_2_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"2" }; - const add_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"1" }; - const sub_len: Adjust = .{ .sign = .neg, .lhs = .len, .op = .mul, .rhs = .@"1" }; - const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" }; - const add_elem_size: Adjust = .{ .sign = .pos, .lhs = .elem_size, .op = .mul, .rhs = .@"1" }; - const add_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .elem_size, .op = .div, .rhs = .@"8" }; - const sub_elem_size_div_8: Adjust = .{ .sign = .neg, .lhs = .elem_size, .op = .div, .rhs = .@"8" }; - const sub_elem_size_div_4: Adjust = .{ .sign = .neg, .lhs = .elem_size, .op = .div, .rhs = .@"4" }; - const add_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; - const add_2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"2" }; - const add_4_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"4" }; - const add_8_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"8" }; - const add_src0_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .div, .rhs = .@"8" }; - const sub_src0_elem_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .div, .rhs = .@"8" }; - const sub_src0_elem_size: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; - const add_src0_elem_size_mul_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; - const sub_src0_elem_size_mul_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; - const add_vector_index: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .mul, .rhs = .@"1" }; - const add_vector_index_rem_32: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .rem_8_mul, .rhs = .@"4" }; - const add_vector_index_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .div_8_down, .rhs = .@"4" }; - const add_dst0_elem_size: Adjust = .{ .sign = .pos, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; - const sub_dst0_elem_size: Adjust = .{ .sign = .neg, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; - const add_src1_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .div_8_down, .rhs = .@"4" }; - const add_src1: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .mul, .rhs = .@"1" }; - const add_src1_rem_32: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"4" }; - const add_src1_rem_64: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"8" }; - const add_src1_sub_bit_size: Adjust = .{ .sign = .pos, .lhs = .src1_sub_bit_size, .op = .mul, .rhs = .@"1" }; - const add_log2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .log2_src0_elem_size, .op = .mul, .rhs = .@"1" }; - const elem_mask: Adjust = .{ .sign = .pos, .lhs = .elem_mask, .op = .mul, .rhs = .@"1" }; - const add_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"1" }; - const add_2_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"2" }; - const sub_smin: Adjust = .{ .sign = .neg, .lhs = .smin, .op = .mul, .rhs = .@"1" }; - const add_smax: Adjust = .{ .sign = .pos, .lhs = .smax, .op = .mul, .rhs = .@"1" }; - const add_umax: Adjust = .{ .sign = .pos, .lhs = .umax, .op = .mul, .rhs = .@"1" }; - const sub_umax: Adjust = .{ .sign = .neg, .lhs = .umax, .op = .mul, .rhs = .@"1" }; - const add_smax_shr_src1: Adjust = .{ .sign = .pos, .lhs = .smax_shr_src1, .op = .mul, .rhs = .@"1" }; - const add_umax_shr_src1: Adjust = .{ .sign = .pos, .lhs = .umax_shr_src1, .op = .mul, .rhs = .@"1" }; - const repeat: Adjust = .{ .sign = .pos, .lhs = .repeat, .op = .mul, .rhs = .@"1" }; - }; - const Ref = enum(u5) { - tmp0, - tmp1, - tmp2, - tmp3, - tmp4, - tmp5, - tmp6, - tmp7, - tmp8, - tmp9, - tmp10, - dst0, - dst1, - src0, - src1, - src2, - none, - - const Sized = packed struct(u9) { - ref: Ref, - size: Memory.Size, - - const none: Sized = .{ .ref = .none, .size = .none }; - - const tmp0: Sized = .{ .ref = .tmp0, .size = .none }; - const tmp0l: Sized = .{ .ref = .tmp0, .size = .low_byte }; - const tmp0h: Sized = .{ .ref = .tmp0, .size = .high_byte }; - const tmp0b: Sized = .{ .ref = .tmp0, .size = .byte }; - const tmp0w: Sized = .{ .ref = .tmp0, .size = .word }; - const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword }; - const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr }; - const tmp0g: Sized = .{ .ref = .tmp0, .size = .gpr }; - const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword }; - const tmp0t: Sized = .{ .ref = .tmp0, .size = .tbyte }; - const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword }; - const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword }; - - const tmp1: Sized = .{ .ref = .tmp1, .size = .none }; - const tmp1l: Sized = .{ .ref = .tmp1, .size = .low_byte }; - const tmp1h: Sized = .{ .ref = .tmp1, .size = .high_byte }; - const tmp1b: Sized = .{ .ref = .tmp1, .size = .byte }; - const tmp1w: Sized = .{ .ref = .tmp1, .size = .word }; - const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword }; - const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr }; - const tmp1g: Sized = .{ .ref = .tmp1, .size = .gpr }; - const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword }; - const tmp1t: Sized = .{ .ref = .tmp1, .size = .tbyte }; - const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword }; - const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword }; - - const tmp2: Sized = .{ .ref = .tmp2, .size = .none }; - const tmp2l: Sized = .{ .ref = .tmp2, .size = .low_byte }; - const tmp2h: Sized = .{ .ref = .tmp2, .size = .high_byte }; - const tmp2b: Sized = .{ .ref = .tmp2, .size = .byte }; - const tmp2w: Sized = .{ .ref = .tmp2, .size = .word }; - const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword }; - const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr }; - const tmp2g: Sized = .{ .ref = .tmp2, .size = .gpr }; - const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword }; - const tmp2t: Sized = .{ .ref = .tmp2, .size = .tbyte }; - const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword }; - const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword }; - - const tmp3: Sized = .{ .ref = .tmp3, .size = .none }; - const tmp3l: Sized = .{ .ref = .tmp3, .size = .low_byte }; - const tmp3h: Sized = .{ .ref = .tmp3, .size = .high_byte }; - const tmp3b: Sized = .{ .ref = .tmp3, .size = .byte }; - const tmp3w: Sized = .{ .ref = .tmp3, .size = .word }; - const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword }; - const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr }; - const tmp3g: Sized = .{ .ref = .tmp3, .size = .gpr }; - const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword }; - const tmp3t: Sized = .{ .ref = .tmp3, .size = .tbyte }; - const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword }; - const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword }; - - const tmp4: Sized = .{ .ref = .tmp4, .size = .none }; - const tmp4l: Sized = .{ .ref = .tmp4, .size = .low_byte }; - const tmp4h: Sized = .{ .ref = .tmp4, .size = .high_byte }; - const tmp4b: Sized = .{ .ref = .tmp4, .size = .byte }; - const tmp4w: Sized = .{ .ref = .tmp4, .size = .word }; - const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword }; - const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr }; - const tmp4g: Sized = .{ .ref = .tmp4, .size = .gpr }; - const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword }; - const tmp4t: Sized = .{ .ref = .tmp4, .size = .tbyte }; - const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword }; - const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword }; - - const tmp5: Sized = .{ .ref = .tmp5, .size = .none }; - const tmp5l: Sized = .{ .ref = .tmp5, .size = .low_byte }; - const tmp5h: Sized = .{ .ref = .tmp5, .size = .high_byte }; - const tmp5b: Sized = .{ .ref = .tmp5, .size = .byte }; - const tmp5w: Sized = .{ .ref = .tmp5, .size = .word }; - const tmp5d: Sized = .{ .ref = .tmp5, .size = .dword }; - const tmp5p: Sized = .{ .ref = .tmp5, .size = .ptr }; - const tmp5g: Sized = .{ .ref = .tmp5, .size = .gpr }; - const tmp5q: Sized = .{ .ref = .tmp5, .size = .qword }; - const tmp5t: Sized = .{ .ref = .tmp5, .size = .tbyte }; - const tmp5x: Sized = .{ .ref = .tmp5, .size = .xword }; - const tmp5y: Sized = .{ .ref = .tmp5, .size = .yword }; - - const tmp6: Sized = .{ .ref = .tmp6, .size = .none }; - const tmp6l: Sized = .{ .ref = .tmp6, .size = .low_byte }; - const tmp6h: Sized = .{ .ref = .tmp6, .size = .high_byte }; - const tmp6b: Sized = .{ .ref = .tmp6, .size = .byte }; - const tmp6w: Sized = .{ .ref = .tmp6, .size = .word }; - const tmp6d: Sized = .{ .ref = .tmp6, .size = .dword }; - const tmp6p: Sized = .{ .ref = .tmp6, .size = .ptr }; - const tmp6g: Sized = .{ .ref = .tmp6, .size = .gpr }; - const tmp6q: Sized = .{ .ref = .tmp6, .size = .qword }; - const tmp6t: Sized = .{ .ref = .tmp6, .size = .tbyte }; - const tmp6x: Sized = .{ .ref = .tmp6, .size = .xword }; - const tmp6y: Sized = .{ .ref = .tmp6, .size = .yword }; - - const tmp7: Sized = .{ .ref = .tmp7, .size = .none }; - const tmp7l: Sized = .{ .ref = .tmp7, .size = .low_byte }; - const tmp7h: Sized = .{ .ref = .tmp7, .size = .high_byte }; - const tmp7b: Sized = .{ .ref = .tmp7, .size = .byte }; - const tmp7w: Sized = .{ .ref = .tmp7, .size = .word }; - const tmp7d: Sized = .{ .ref = .tmp7, .size = .dword }; - const tmp7p: Sized = .{ .ref = .tmp7, .size = .ptr }; - const tmp7g: Sized = .{ .ref = .tmp7, .size = .gpr }; - const tmp7q: Sized = .{ .ref = .tmp7, .size = .qword }; - const tmp7t: Sized = .{ .ref = .tmp7, .size = .tbyte }; - const tmp7x: Sized = .{ .ref = .tmp7, .size = .xword }; - const tmp7y: Sized = .{ .ref = .tmp7, .size = .yword }; - - const tmp8: Sized = .{ .ref = .tmp8, .size = .none }; - const tmp8l: Sized = .{ .ref = .tmp8, .size = .low_byte }; - const tmp8h: Sized = .{ .ref = .tmp8, .size = .high_byte }; - const tmp8b: Sized = .{ .ref = .tmp8, .size = .byte }; - const tmp8w: Sized = .{ .ref = .tmp8, .size = .word }; - const tmp8d: Sized = .{ .ref = .tmp8, .size = .dword }; - const tmp8p: Sized = .{ .ref = .tmp8, .size = .ptr }; - const tmp8g: Sized = .{ .ref = .tmp8, .size = .gpr }; - const tmp8q: Sized = .{ .ref = .tmp8, .size = .qword }; - const tmp8t: Sized = .{ .ref = .tmp8, .size = .tbyte }; - const tmp8x: Sized = .{ .ref = .tmp8, .size = .xword }; - const tmp8y: Sized = .{ .ref = .tmp8, .size = .yword }; - - const tmp9: Sized = .{ .ref = .tmp9, .size = .none }; - const tmp9l: Sized = .{ .ref = .tmp9, .size = .low_byte }; - const tmp9h: Sized = .{ .ref = .tmp9, .size = .high_byte }; - const tmp9b: Sized = .{ .ref = .tmp9, .size = .byte }; - const tmp9w: Sized = .{ .ref = .tmp9, .size = .word }; - const tmp9d: Sized = .{ .ref = .tmp9, .size = .dword }; - const tmp9p: Sized = .{ .ref = .tmp9, .size = .ptr }; - const tmp9g: Sized = .{ .ref = .tmp9, .size = .gpr }; - const tmp9q: Sized = .{ .ref = .tmp9, .size = .qword }; - const tmp9t: Sized = .{ .ref = .tmp9, .size = .tbyte }; - const tmp9x: Sized = .{ .ref = .tmp9, .size = .xword }; - const tmp9y: Sized = .{ .ref = .tmp9, .size = .yword }; - - const tmp10: Sized = .{ .ref = .tmp10, .size = .none }; - const tmp10l: Sized = .{ .ref = .tmp10, .size = .low_byte }; - const tmp10h: Sized = .{ .ref = .tmp10, .size = .high_byte }; - const tmp10b: Sized = .{ .ref = .tmp10, .size = .byte }; - const tmp10w: Sized = .{ .ref = .tmp10, .size = .word }; - const tmp10d: Sized = .{ .ref = .tmp10, .size = .dword }; - const tmp10p: Sized = .{ .ref = .tmp10, .size = .ptr }; - const tmp10g: Sized = .{ .ref = .tmp10, .size = .gpr }; - const tmp10q: Sized = .{ .ref = .tmp10, .size = .qword }; - const tmp10t: Sized = .{ .ref = .tmp10, .size = .tbyte }; - const tmp10x: Sized = .{ .ref = .tmp10, .size = .xword }; - const tmp10y: Sized = .{ .ref = .tmp10, .size = .yword }; - - const dst0: Sized = .{ .ref = .dst0, .size = .none }; - const dst0l: Sized = .{ .ref = .dst0, .size = .low_byte }; - const dst0h: Sized = .{ .ref = .dst0, .size = .high_byte }; - const dst0b: Sized = .{ .ref = .dst0, .size = .byte }; - const dst0w: Sized = .{ .ref = .dst0, .size = .word }; - const dst0d: Sized = .{ .ref = .dst0, .size = .dword }; - const dst0p: Sized = .{ .ref = .dst0, .size = .ptr }; - const dst0g: Sized = .{ .ref = .dst0, .size = .gpr }; - const dst0q: Sized = .{ .ref = .dst0, .size = .qword }; - const dst0t: Sized = .{ .ref = .dst0, .size = .tbyte }; - const dst0x: Sized = .{ .ref = .dst0, .size = .xword }; - const dst0y: Sized = .{ .ref = .dst0, .size = .yword }; - - const dst1: Sized = .{ .ref = .dst1, .size = .none }; - const dst1l: Sized = .{ .ref = .dst1, .size = .low_byte }; - const dst1h: Sized = .{ .ref = .dst1, .size = .high_byte }; - const dst1b: Sized = .{ .ref = .dst1, .size = .byte }; - const dst1w: Sized = .{ .ref = .dst1, .size = .word }; - const dst1d: Sized = .{ .ref = .dst1, .size = .dword }; - const dst1p: Sized = .{ .ref = .dst1, .size = .ptr }; - const dst1g: Sized = .{ .ref = .dst1, .size = .gpr }; - const dst1q: Sized = .{ .ref = .dst1, .size = .qword }; - const dst1t: Sized = .{ .ref = .dst1, .size = .tbyte }; - const dst1x: Sized = .{ .ref = .dst1, .size = .xword }; - const dst1y: Sized = .{ .ref = .dst1, .size = .yword }; - - const src0: Sized = .{ .ref = .src0, .size = .none }; - const src0l: Sized = .{ .ref = .src0, .size = .low_byte }; - const src0h: Sized = .{ .ref = .src0, .size = .high_byte }; - const src0b: Sized = .{ .ref = .src0, .size = .byte }; - const src0w: Sized = .{ .ref = .src0, .size = .word }; - const src0d: Sized = .{ .ref = .src0, .size = .dword }; - const src0p: Sized = .{ .ref = .src0, .size = .ptr }; - const src0g: Sized = .{ .ref = .src0, .size = .gpr }; - const src0q: Sized = .{ .ref = .src0, .size = .qword }; - const src0t: Sized = .{ .ref = .src0, .size = .tbyte }; - const src0x: Sized = .{ .ref = .src0, .size = .xword }; - const src0y: Sized = .{ .ref = .src0, .size = .yword }; - - const src1: Sized = .{ .ref = .src1, .size = .none }; - const src1l: Sized = .{ .ref = .src1, .size = .low_byte }; - const src1h: Sized = .{ .ref = .src1, .size = .high_byte }; - const src1b: Sized = .{ .ref = .src1, .size = .byte }; - const src1w: Sized = .{ .ref = .src1, .size = .word }; - const src1d: Sized = .{ .ref = .src1, .size = .dword }; - const src1p: Sized = .{ .ref = .src1, .size = .ptr }; - const src1g: Sized = .{ .ref = .src1, .size = .gpr }; - const src1q: Sized = .{ .ref = .src1, .size = .qword }; - const src1t: Sized = .{ .ref = .src1, .size = .tbyte }; - const src1x: Sized = .{ .ref = .src1, .size = .xword }; - const src1y: Sized = .{ .ref = .src1, .size = .yword }; - - const src2: Sized = .{ .ref = .src2, .size = .none }; - const src2l: Sized = .{ .ref = .src2, .size = .low_byte }; - const src2h: Sized = .{ .ref = .src2, .size = .high_byte }; - const src2b: Sized = .{ .ref = .src2, .size = .byte }; - const src2w: Sized = .{ .ref = .src2, .size = .word }; - const src2d: Sized = .{ .ref = .src2, .size = .dword }; - const src2p: Sized = .{ .ref = .src2, .size = .ptr }; - const src2g: Sized = .{ .ref = .src2, .size = .gpr }; - const src2q: Sized = .{ .ref = .src2, .size = .qword }; - const src2t: Sized = .{ .ref = .src2, .size = .tbyte }; - const src2x: Sized = .{ .ref = .src2, .size = .xword }; - const src2y: Sized = .{ .ref = .src2, .size = .yword }; - }; - - fn typeOf(ref: Ref, s: *const Select) Type { - return s.types[@intFromEnum(ref)]; - } - - fn tempOf(ref: Ref, s: *const Select) Temp { - return s.temps[@intFromEnum(ref)]; - } - - fn valueOf(ref: Ref, s: *const Select) MCValue { - return switch (ref) { - .none => .none, - else => s.temps[@intFromEnum(ref)].tracking(s.cg).short, - }; - } - }; - - const @"_": Select.Operand = .{ .flags = .{ .tag = .none } }; - - const @"0b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 0) }; - const @"0f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 0) }; - const @"1b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 1) }; - const @"1f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 1) }; - const @"2b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 2) }; - const @"2f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 2) }; - const @"3b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 3) }; - const @"3f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 3) }; - const @"4b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 4) }; - const @"4f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 4) }; - - const tmp0l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0l } }; - const tmp0h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0h } }; - const tmp0b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0b } }; - const tmp0w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0w } }; - const tmp0d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0d } }; - const tmp0p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0p } }; - const tmp0g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0g } }; - const tmp0q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0q } }; - const tmp0t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0t } }; - const tmp0x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0x } }; - const tmp0y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0y } }; - const tmp0w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0w }, .imm = 1 }; - const tmp0d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0d }, .imm = 1 }; - const tmp0p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0p }, .imm = 1 }; - const tmp0q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0q }, .imm = 1 }; - const tmp0w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0w }, .imm = 2 }; - const tmp0d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0d }, .imm = 2 }; - const tmp0p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0p }, .imm = 2 }; - const tmp0q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0q }, .imm = 2 }; - - const tmp1l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1l } }; - const tmp1h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1h } }; - const tmp1b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1b } }; - const tmp1w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1w } }; - const tmp1d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1d } }; - const tmp1p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1p } }; - const tmp1g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1g } }; - const tmp1q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1q } }; - const tmp1t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1t } }; - const tmp1x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1x } }; - const tmp1y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1y } }; - const tmp1w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1w }, .imm = 1 }; - const tmp1d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1d }, .imm = 1 }; - const tmp1p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1p }, .imm = 1 }; - const tmp1q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1q }, .imm = 1 }; - const tmp1w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1w }, .imm = 2 }; - const tmp1d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1d }, .imm = 2 }; - const tmp1p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1p }, .imm = 2 }; - const tmp1q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1q }, .imm = 2 }; - - const tmp2l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2l } }; - const tmp2h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2h } }; - const tmp2b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2b } }; - const tmp2w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2w } }; - const tmp2d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2d } }; - const tmp2p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2p } }; - const tmp2g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2g } }; - const tmp2q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2q } }; - const tmp2t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2t } }; - const tmp2x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2x } }; - const tmp2y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2y } }; - const tmp2w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2w }, .imm = 1 }; - const tmp2d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2d }, .imm = 1 }; - const tmp2p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2p }, .imm = 1 }; - const tmp2q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2q }, .imm = 1 }; - const tmp2w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2w }, .imm = 2 }; - const tmp2d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2d }, .imm = 2 }; - const tmp2p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2p }, .imm = 2 }; - const tmp2q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2q }, .imm = 2 }; - - const tmp3l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3l } }; - const tmp3h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3h } }; - const tmp3b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3b } }; - const tmp3w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3w } }; - const tmp3d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3d } }; - const tmp3p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3p } }; - const tmp3g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3g } }; - const tmp3q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3q } }; - const tmp3t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3t } }; - const tmp3x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3x } }; - const tmp3y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3y } }; - const tmp3w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3w }, .imm = 1 }; - const tmp3d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3d }, .imm = 1 }; - const tmp3p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3p }, .imm = 1 }; - const tmp3q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3q }, .imm = 1 }; - const tmp3w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3w }, .imm = 2 }; - const tmp3d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3d }, .imm = 2 }; - const tmp3p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3p }, .imm = 2 }; - const tmp3q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3q }, .imm = 2 }; - - const tmp4l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4l } }; - const tmp4h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4h } }; - const tmp4b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4b } }; - const tmp4w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4w } }; - const tmp4d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4d } }; - const tmp4p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4p } }; - const tmp4g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4g } }; - const tmp4q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4q } }; - const tmp4t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4t } }; - const tmp4x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4x } }; - const tmp4y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4y } }; - const tmp4w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4w }, .imm = 1 }; - const tmp4d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4d }, .imm = 1 }; - const tmp4p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4p }, .imm = 1 }; - const tmp4q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4q }, .imm = 1 }; - const tmp4w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4w }, .imm = 2 }; - const tmp4d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4d }, .imm = 2 }; - const tmp4p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4p }, .imm = 2 }; - const tmp4q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4q }, .imm = 2 }; - - const tmp5l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5l } }; - const tmp5h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5h } }; - const tmp5b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5b } }; - const tmp5w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5w } }; - const tmp5d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5d } }; - const tmp5p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5p } }; - const tmp5g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5g } }; - const tmp5q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5q } }; - const tmp5t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5t } }; - const tmp5x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5x } }; - const tmp5y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5y } }; - const tmp5w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5w }, .imm = 1 }; - const tmp5d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5d }, .imm = 1 }; - const tmp5p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5p }, .imm = 1 }; - const tmp5q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5q }, .imm = 1 }; - const tmp5w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5w }, .imm = 2 }; - const tmp5d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5d }, .imm = 2 }; - const tmp5p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5p }, .imm = 2 }; - const tmp5q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5q }, .imm = 2 }; - - const tmp6l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6l } }; - const tmp6h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6h } }; - const tmp6b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6b } }; - const tmp6w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6w } }; - const tmp6d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6d } }; - const tmp6p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6p } }; - const tmp6g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6g } }; - const tmp6q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6q } }; - const tmp6t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6t } }; - const tmp6x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6x } }; - const tmp6y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6y } }; - const tmp6w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6w }, .imm = 1 }; - const tmp6d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6d }, .imm = 1 }; - const tmp6p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6p }, .imm = 1 }; - const tmp6q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6q }, .imm = 1 }; - const tmp6w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6w }, .imm = 2 }; - const tmp6d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6d }, .imm = 2 }; - const tmp6p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6p }, .imm = 2 }; - const tmp6q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6q }, .imm = 2 }; - - const tmp7l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7l } }; - const tmp7h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7h } }; - const tmp7b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7b } }; - const tmp7w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7w } }; - const tmp7d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7d } }; - const tmp7p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7p } }; - const tmp7g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7g } }; - const tmp7q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7q } }; - const tmp7t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7t } }; - const tmp7x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7x } }; - const tmp7y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7y } }; - const tmp7w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7w }, .imm = 1 }; - const tmp7d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7d }, .imm = 1 }; - const tmp7p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7p }, .imm = 1 }; - const tmp7q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7q }, .imm = 1 }; - const tmp7w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7w }, .imm = 2 }; - const tmp7d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7d }, .imm = 2 }; - const tmp7p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7p }, .imm = 2 }; - const tmp7q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7q }, .imm = 2 }; - - const tmp8l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8l } }; - const tmp8h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8h } }; - const tmp8b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8b } }; - const tmp8w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8w } }; - const tmp8d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8d } }; - const tmp8p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8p } }; - const tmp8g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8g } }; - const tmp8q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8q } }; - const tmp8t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8t } }; - const tmp8x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8x } }; - const tmp8y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8y } }; - const tmp8w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8w }, .imm = 1 }; - const tmp8d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8d }, .imm = 1 }; - const tmp8p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8p }, .imm = 1 }; - const tmp8q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8q }, .imm = 1 }; - const tmp8w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8w }, .imm = 2 }; - const tmp8d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8d }, .imm = 2 }; - const tmp8p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8p }, .imm = 2 }; - const tmp8q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8q }, .imm = 2 }; - - const tmp9l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9l } }; - const tmp9h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9h } }; - const tmp9b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9b } }; - const tmp9w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9w } }; - const tmp9d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9d } }; - const tmp9p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9p } }; - const tmp9g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9g } }; - const tmp9q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9q } }; - const tmp9t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9t } }; - const tmp9x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9x } }; - const tmp9y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9y } }; - const tmp9w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9w }, .imm = 1 }; - const tmp9d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9d }, .imm = 1 }; - const tmp9p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9p }, .imm = 1 }; - const tmp9q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9q }, .imm = 1 }; - const tmp9w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9w }, .imm = 2 }; - const tmp9d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9d }, .imm = 2 }; - const tmp9p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9p }, .imm = 2 }; - const tmp9q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9q }, .imm = 2 }; - - const tmp10l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10l } }; - const tmp10h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10h } }; - const tmp10b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10b } }; - const tmp10w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10w } }; - const tmp10d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10d } }; - const tmp10p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10p } }; - const tmp10g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10g } }; - const tmp10q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10q } }; - const tmp10t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10t } }; - const tmp10x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10x } }; - const tmp10y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10y } }; - const tmp10w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10w }, .imm = 1 }; - const tmp10d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10d }, .imm = 1 }; - const tmp10p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10p }, .imm = 1 }; - const tmp10q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10q }, .imm = 1 }; - const tmp10w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10w }, .imm = 2 }; - const tmp10d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10d }, .imm = 2 }; - const tmp10p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10p }, .imm = 2 }; - const tmp10q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10q }, .imm = 2 }; - - const dst0l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0l } }; - const dst0h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0h } }; - const dst0b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0b } }; - const dst0w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0w } }; - const dst0d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0d } }; - const dst0p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0p } }; - const dst0g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0g } }; - const dst0q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0q } }; - const dst0t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0t } }; - const dst0x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0x } }; - const dst0y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0y } }; - const dst0w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0w }, .imm = 1 }; - const dst0d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0d }, .imm = 1 }; - const dst0p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0p }, .imm = 1 }; - const dst0q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0q }, .imm = 1 }; - const dst0w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0w }, .imm = 2 }; - const dst0d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0d }, .imm = 2 }; - const dst0p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0p }, .imm = 2 }; - const dst0q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0q }, .imm = 2 }; - - const dst1l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1l } }; - const dst1h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1h } }; - const dst1b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1b } }; - const dst1w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1w } }; - const dst1d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1d } }; - const dst1p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1p } }; - const dst1g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1g } }; - const dst1q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1q } }; - const dst1t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1t } }; - const dst1x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1x } }; - const dst1y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1y } }; - const dst1w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1w }, .imm = 1 }; - const dst1d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1d }, .imm = 1 }; - const dst1p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1p }, .imm = 1 }; - const dst1q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1q }, .imm = 1 }; - const dst1w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1w }, .imm = 2 }; - const dst1d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1d }, .imm = 2 }; - const dst1p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1p }, .imm = 2 }; - const dst1q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1q }, .imm = 2 }; - - const src0l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0l } }; - const src0h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0h } }; - const src0b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0b } }; - const src0w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0w } }; - const src0d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0d } }; - const src0p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0p } }; - const src0g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0g } }; - const src0q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0q } }; - const src0t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0t } }; - const src0x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0x } }; - const src0y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0y } }; - const src0w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0w }, .imm = 1 }; - const src0d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0d }, .imm = 1 }; - const src0p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0p }, .imm = 1 }; - const src0q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0q }, .imm = 1 }; - const src0w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0w }, .imm = 2 }; - const src0d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0d }, .imm = 2 }; - const src0p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0p }, .imm = 2 }; - const src0q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0q }, .imm = 2 }; - - const src1l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1l } }; - const src1h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1h } }; - const src1b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1b } }; - const src1w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1w } }; - const src1d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1d } }; - const src1p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1p } }; - const src1g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1g } }; - const src1q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1q } }; - const src1t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1t } }; - const src1x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1x } }; - const src1y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1y } }; - const src1w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1w }, .imm = 1 }; - const src1d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1d }, .imm = 1 }; - const src1p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1p }, .imm = 1 }; - const src1q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1q }, .imm = 1 }; - const src1w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1w }, .imm = 2 }; - const src1d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1d }, .imm = 2 }; - const src1p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1p }, .imm = 2 }; - const src1q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1q }, .imm = 2 }; - - const src2l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2l } }; - const src2h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2h } }; - const src2b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2b } }; - const src2w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2w } }; - const src2d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2d } }; - const src2p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2p } }; - const src2g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2g } }; - const src2q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2q } }; - const src2t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2t } }; - const src2x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2x } }; - const src2y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2y } }; - const src2w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2w }, .imm = 1 }; - const src2d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2d }, .imm = 1 }; - const src2p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2p }, .imm = 1 }; - const src2q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2q }, .imm = 1 }; - const src2w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2w }, .imm = 2 }; - const src2d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2d }, .imm = 2 }; - const src2p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2p }, .imm = 2 }; - const src2q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2q }, .imm = 2 }; - - fn si(imm: i32) Select.Operand { - return .{ .flags = .{ .tag = .simm }, .imm = imm }; - } - fn sa(base: Ref.Sized, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base } }; - } - fn sa2(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base, .index = .{ .ref = index } } }; - } - fn sia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base }, .imm = imm }; - } - fn sia2(imm: i32, base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base, .index = .{ .ref = index } }, .imm = imm }; - } - fn ui(imm: u32) Select.Operand { - return .{ .flags = .{ .tag = .uimm }, .imm = @bitCast(imm) }; - } - fn ua(base: Ref.Sized, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .uimm, .adjust = adjust, .base = base } }; - } - fn uia(imm: u32, base: Ref.Sized, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .uimm, .adjust = adjust, .base = base }, .imm = @bitCast(imm) }; - } - fn uas(base: Ref.Sized, scale: Memory.Scale, adjust: Adjust) Select.Operand { - return .{ .flags = .{ .tag = .uimm, .adjust = adjust, .base = base, .index = .{ .scale = scale } } }; - } - - fn rm(mode: bits.RoundMode) Select.Operand { - return .{ .flags = .{ .tag = .uimm }, .imm = @intCast(mode.imm().unsigned) }; - } - fn sp(pred: bits.SseFloatPredicate) Select.Operand { - return .{ .flags = .{ .tag = .uimm }, .imm = @intCast(pred.imm().unsigned) }; - } - fn vp(pred: bits.VexFloatPredicate) Select.Operand { - return .{ .flags = .{ .tag = .uimm }, .imm = @intCast(pred.imm().unsigned) }; - } - - fn lea(base: Ref.Sized) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .base = base, - }, - }; - } - fn leaa(base: Ref.Sized, adjust: Adjust) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .adjust = adjust, - .base = base, - }, - }; - } - fn leaad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .adjust = adjust, - .base = base, - }, - .imm = disp, - }; - } - fn lead(base: Ref.Sized, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .base = base, - }, - .imm = disp, - }; - } - fn leai(base: Ref.Sized, index: Ref) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .base = base, - .index = .{ .ref = index }, - }, - }; - } - fn leaia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .adjust = adjust, - .base = base, - .index = .{ .ref = index }, - }, - }; - } - fn leaiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .adjust = adjust, - .base = base, - .index = .{ .ref = index }, - }, - .imm = disp, - }; - } - fn leaid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .base = base, - .index = .{ .ref = index }, - }, - .imm = disp, - }; - } - fn leasi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - }; - } - fn leasia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .adjust = adjust, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - }; - } - fn leasid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - .imm = disp, - }; - } - fn leasiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .lea, - .adjust = adjust, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - .imm = disp, - }; - } - - fn mem(base: Ref.Sized) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .base = base, - }, - }; - } - fn memd(base: Ref.Sized, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .base = base, - }, - .imm = disp, - }; - } - fn mema(base: Ref.Sized, adjust: Adjust) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .adjust = adjust, - .base = base, - }, - }; - } - fn memad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .adjust = adjust, - .base = base, - }, - .imm = disp, - }; - } - fn memi(base: Ref.Sized, index: Ref) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .base = base, - .index = .{ .ref = index }, - }, - }; - } - fn memia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .adjust = adjust, - .base = base, - .index = .{ .ref = index }, - }, - }; - } - fn memiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .adjust = adjust, - .base = base, - .index = .{ .ref = index }, - }, - .imm = disp, - }; - } - fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .base = base, - .index = .{ .ref = index }, - }, - .imm = disp, - }; - } - fn memsi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - }; - } - fn memsia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .adjust = adjust, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - }; - } - fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - .imm = disp, - }; - } - fn memsiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand { - return .{ - .flags = .{ - .tag = .mem, - .adjust = adjust, - .base = base, - .index = .{ .ref = index, .scale = scale }, - }, - .imm = disp, - }; - } - - fn adjustedImm(op: Select.Operand, comptime SignedImm: type, s: *const Select) SignedImm { - const UnsignedImm = @Type(.{ - .int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits }, - }); - const lhs: SignedImm = lhs: switch (op.flags.adjust.lhs) { - .none => 0, - .ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8), - .ptr_bit_size => s.cg.target.ptrBitWidth(), - .size => @intCast(op.flags.base.ref.typeOf(s).abiSize(s.cg.pt.zcu)), - .src0_size => @intCast(Select.Operand.Ref.src0.typeOf(s).abiSize(s.cg.pt.zcu)), - .dst0_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).abiSize(s.cg.pt.zcu)), - .delta_size => @intCast(@as(SignedImm, @intCast(op.flags.base.ref.typeOf(s).abiSize(s.cg.pt.zcu))) - - @as(SignedImm, @intCast(op.flags.index.ref.typeOf(s).abiSize(s.cg.pt.zcu)))), - .delta_elem_size => @intCast(@as(SignedImm, @intCast(op.flags.base.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))) - - @as(SignedImm, @intCast(op.flags.index.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)))), - .unaligned_size => @intCast(s.cg.unalignedSize(op.flags.base.ref.typeOf(s))), - .unaligned_size_add_elem_size => { - const ty = op.flags.base.ref.typeOf(s); - break :lhs @intCast(s.cg.unalignedSize(ty) + ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)); - }, - .unaligned_size_sub_elem_size => { - const ty = op.flags.base.ref.typeOf(s); - break :lhs @intCast(s.cg.unalignedSize(ty) - ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)); - }, - .unaligned_size_sub_2_elem_size => { - const ty = op.flags.base.ref.typeOf(s); - break :lhs @intCast(s.cg.unalignedSize(ty) - ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * 2); - }, - .bit_size => @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s))), - .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))), - .@"8_size_sub_bit_size" => { - const ty = op.flags.base.ref.typeOf(s); - break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu)); - }, - .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)), - .elem_limbs => @intCast(@divExact( - op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu), - @divExact(op.flags.base.size.bitSize(s.cg.target), 8), - )), - .elem_size => @intCast(op.flags.base.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), - .src0_elem_size => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), - .dst0_elem_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), - .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * - Select.Operand.Ref.src1.valueOf(s).immediate), - .vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) { - .none, .runtime => unreachable, - else => |vector_index| @intFromEnum(vector_index), - }, - .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), - .src1_sub_bit_size => @as(SignedImm, @intCast(Select.Operand.Ref.src1.valueOf(s).immediate)) - - @as(SignedImm, @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)))), - .log2_src0_elem_size => @intCast(std.math.log2(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))), - .elem_mask => @as(u8, std.math.maxInt(u8)) >> @intCast( - 8 - ((s.cg.unalignedSize(op.flags.base.ref.typeOf(s)) - 1) % - @divExact(op.flags.base.size.bitSize(s.cg.target), 8) + 1 >> - op.flags.index.scale.toLog2()), - ), - inline .smin, .smax, .umax, .smin_shr_src1, .smax_shr_src1, .umax_shr_src1 => |adjust| switch (op.flags.base.size) { - else => unreachable, - inline .none, .byte, .word, .dword, .qword => |size| { - const Imm = switch (adjust) { - else => comptime unreachable, - .smin, .smax, .smin_shr_src1, .smax_shr_src1 => SignedImm, - .umax, .umax_shr_src1 => UnsignedImm, - }; - const RefImm = switch (size) { - else => comptime unreachable, - .none => Imm, - .byte, .word, .dword, .qword => @Type(comptime .{ .int = .{ - .signedness = @typeInfo(Imm).int.signedness, - .bits = size.bitSize(undefined), - } }), - }; - break :lhs @bitCast(@as(Imm, @intCast(@as(RefImm, switch (adjust) { - else => comptime unreachable, - .smin, .smin_shr_src1 => std.math.minInt, - .smax, .umax, .smax_shr_src1, .umax_shr_src1 => std.math.maxInt, - }(RefImm)) >> @truncate(switch (adjust) { - else => comptime unreachable, - .smin, .smax, .umax => 0, - .smin_shr_src1, .smax_shr_src1, .umax_shr_src1 => Select.Operand.Ref.src1.valueOf(s).immediate, - } -% s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)))))); - }, - }, - .repeat => switch (SignedImm) { - else => unreachable, - i64 => return @as(i64, op.imm) << 32 | @as(u32, @bitCast(op.imm)), - }, - }; - const rhs = op.flags.adjust.rhs.toLog2(); - const op_res = op_res: switch (op.flags.adjust.op) { - .mul => { - const op_res = @shlWithOverflow(lhs, rhs); - assert(op_res[1] == 0); - break :op_res op_res[0]; - }, - .div => @shrExact(lhs, rhs), - .div_8_down => lhs >> 3 & @as(SignedImm, -1) << rhs, - .rem_8_mul => lhs & (@as(SignedImm, 1) << @intCast(@as(u3, 3) + rhs)) - 1, - }; - const disp: SignedImm = @bitCast(@as(UnsignedImm, @as(u32, @bitCast(op.imm)))); - return switch (op.flags.adjust.sign) { - .neg => disp - op_res, - .pos => disp + op_res, - }; - } - - fn lower(op: Select.Operand, s: *Select) InnerError!CodeGen.Operand { - return switch (op.flags.tag) { - .none => .none, - .label => if (op.imm < 0) - .{ .inst = s.labels[@intCast(~op.imm)].backward.? } - else for (&s.labels[@intCast(op.imm)].forward) |*label| { - if (label.*) |_| continue; - label.* = @intCast(s.cg.mir_instructions.len); - break .{ .inst = undefined }; - } else unreachable, - .ref => switch (op.imm) { - else => unreachable, - 0 => switch (op.flags.base.ref.valueOf(s)) { - .none, .unreach, .dead, .undef => unreachable, - .immediate => |imm| .{ .imm = switch (op.flags.base.size) { - .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))), - .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))), - .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))), - .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm), - else => unreachable, - } }, - .eflags => unreachable, - .register => |reg| .{ .reg = s.lowerReg(reg.toSize(op.flags.base.size, s.cg.target)) }, - .register_pair, .register_triple, .register_quadruple, .register_offset, .register_overflow => unreachable, - .register_mask => |reg_mask| .{ .reg = s.lowerReg(reg_mask.reg.toSize(op.flags.base.size, s.cg.target)) }, - .lea_nav => |nav| .{ .imm = .{ .nav = .{ .index = nav } } }, - .lea_uav => |uav| .{ .imm = .{ .uav = uav } }, - .lea_lazy_sym => |lazy_sym| .{ .imm = .{ .lazy_sym = lazy_sym } }, - .lea_extern_func => |extern_func| .{ .imm = .{ .extern_func = extern_func } }, - else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.flags.base.size }) }, - .lea_frame, .elementwise_args, .reserved_frame, .air_ref => unreachable, - }, - 1...2 => |imm| switch (op.flags.base.ref.valueOf(s)) { - inline .register_pair, .register_triple, .register_quadruple => |regs| .{ - .reg = s.lowerReg(regs[@intCast(imm - 1)].toSize(op.flags.base.size, s.cg.target)), - }, - else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ - .size = op.flags.base.size, - .disp = @as(i32, @intCast(@divExact(op.flags.base.size.bitSize(s.cg.target), 8))) * (imm - 1), - }) }, - }, - }, - .simm => .{ .imm = .s(op.adjustedImm(i32, s)) }, - .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) }, - .lea => .{ .mem = try op.flags.base.ref.valueOf(s).deref().mem(s.cg, .{ - .size = op.flags.base.size, - .index = switch (op.flags.index.ref.valueOf(s)) { - else => unreachable, - .none => .none, - .register => |index_reg| index_reg.toSize(.ptr, s.cg.target), - }, - .scale = op.flags.index.scale, - .disp = op.adjustedImm(i32, s) + switch (op.flags.index.ref.valueOf(s)) { - else => unreachable, - .none, .register, .lea_nav, .lea_uav, .lea_lazy_sym, .lea_extern_func => 0, - .register_offset => |base_reg_off| base_reg_off.off, - }, - }) }, - .mem => .{ .mem = try op.flags.base.ref.valueOf(s).mem(s.cg, .{ - .size = op.flags.base.size, - .index = switch (op.flags.index.ref.valueOf(s)) { - else => unreachable, - .none => .none, - .register => |index_reg| index_reg.toSize(.ptr, s.cg.target), - }, - .scale = op.flags.index.scale, - .disp = op.adjustedImm(i32, s), - }) }, - }; - } - }; -}; -fn select( - cg: *CodeGen, - dst_temps: []Temp, - dst_tys: []const Type, - src_temps: []Temp, - cases: []const Select.Case, -) Select.Error!void { - @setEvalBranchQuota(33_500); - cases: for (cases) |case| { - switch (case.required_abi) { - .any => {}, - .gnu => if (switch (cg.target.abi) { - .none, .msvc, .itanium => cg.target.os.tag == .windows, - else => false, - }) continue :cases, - .msvc => if (switch (cg.target.abi) { - .none, .msvc, .itanium => cg.target.os.tag != .windows, - else => true, - }) continue :cases, - } - switch (case.required_cc_abi) { - .any => {}, - .sysv64 => if (cg.target.cCallingConvention().? != .x86_64_sysv) continue :cases, - .win64 => if (cg.target.cCallingConvention().? != .x86_64_win) continue :cases, - } - for (case.required_features) |required_feature| if (required_feature) |feature| if (!cg.hasFeature(feature)) continue :cases; - for (case.src_constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases; - for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases; - if (std.debug.runtime_safety) { - for (case.src_constraints[src_temps.len..]) |src_constraint| assert(src_constraint == .any); - for (case.dst_constraints[dst_temps.len..]) |dst_constraint| assert(dst_constraint == .any); - for (case.dst_temps[dst_temps.len..]) |dst_temp| assert(dst_temp == .unused); - } - patterns: for (case.patterns) |pattern| { - for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; - if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) |src_pattern| assert(src_pattern == .none); - - if (case.call_frame.alignment != .none) { - const frame_allocs_slice = cg.frame_allocs.slice(); - const stack_frame_size = - &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; - stack_frame_size.* = @max(stack_frame_size.*, switch (cg.target.cCallingConvention().?) { - .x86_64_sysv => case.call_frame.size, - .x86_64_win => win64_shadow_space + case.call_frame.size, - else => unreachable, - }); - const stack_frame_align = - &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; - stack_frame_align.* = stack_frame_align.max(case.call_frame.alignment); - } - - var s: Select = .{ - .cg = cg, - .temps = undefined, - .types = undefined, - .labels = @splat(.{ .forward = @splat(null), .backward = null }), - .top = 0, - }; - const s_tmp_types = s.types[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)]; - const s_tmp_temps = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)]; - const s_dst_types = s.types[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; - const s_dst_temps = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; - const s_src_types = s.types[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; - const s_src_temps = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; - - for (s_tmp_types, case.extra_temps) |*ty, spec| ty.* = spec.type; - @memcpy(s_dst_types[0..dst_tys.len], dst_tys); - for (s_src_types[0..src_temps.len], src_temps) |*ty, temp| ty.* = temp.typeOf(cg); - std.mem.swap(Type, &s_src_types[pattern.commute[0]], &s_src_types[pattern.commute[1]]); - - caller_preserved: { - const cc = switch (case.clobbers.caller_preserved) { - .none => break :caller_preserved, - .ccc => cg.target.cCallingConvention().?, - .zigcc => .auto, - }; - assert(case.clobbers.eflags); - const err_ret_trace_reg = if (cc == .auto and cg.pt.zcu.comp.config.any_error_tracing) err_ret_trace_reg: { - const param_gpr = abi.getCAbiIntParamRegs(.auto); - break :err_ret_trace_reg param_gpr[param_gpr.len - 1]; - } else .none; - switch (cc) { - else => unreachable, - inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| skip: { - if (reg == err_ret_trace_reg) break :skip; - const tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse break :skip; - try cg.register_manager.getRegIndex(tracked_index, null); - assert(cg.register_manager.lockRegIndexAssumeUnused(tracked_index).tracked_index == tracked_index); - }, - } - } - - @memcpy(s_src_temps[0..src_temps.len], src_temps); - std.mem.swap(Temp, &s_src_temps[pattern.commute[0]], &s_src_temps[pattern.commute[1]]); - var dst_locks: [s_dst_temps.len][2]?RegisterLock = @splat(@splat(null)); - for (dst_locks[0..dst_temps.len], case.dst_temps[0..dst_temps.len]) |*dst_lock, dst_kind| dst_lock.* = try dst_kind.lock(cg); - var tmp_locks: [s_tmp_temps.len][2]?RegisterLock = @splat(@splat(null)); - for (&tmp_locks, case.extra_temps) |*tmp_lock, tmp_spec| tmp_lock.* = try tmp_spec.kind.lock(cg); - - while (true) for (pattern.src[0..src_temps.len], src_temps) |src_pattern, *src_temp| { - if (try src_pattern.convert(src_temp, cg)) break; - } else break; - var src_locks: [s_src_temps.len][2]?RegisterLock = @splat(@splat(null)); - for (src_locks[0..src_temps.len], src_temps) |*locks, src_temp| { - const regs: [2]Register = switch (src_temp.tracking(cg).short) { - else => continue, - .register => |reg| .{ reg, .none }, - .register_pair => |regs| regs, - }; - for (regs, locks) |reg, *lock| { - if (reg == .none) continue; - lock.* = cg.register_manager.lockRegIndex(RegisterManager.indexOfRegIntoTracked(reg) orelse continue); - } - } - @memcpy(s_src_temps[0..src_temps.len], src_temps); - std.mem.swap(Temp, &s_src_temps[pattern.commute[0]], &s_src_temps[pattern.commute[1]]); - - if (case.clobbers.eflags) try cg.spillEflagsIfOccupied(); - - var tmp_owned: [s_tmp_temps.len]bool = @splat(false); - for (s_tmp_temps, &tmp_owned, case.extra_temps) |*temp, *owned, tmp_spec| temp.*, owned.* = try tmp_spec.create(&s); - for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, tmp_kind| dst_temp.*, _ = try Select.TempSpec.create(.{ .type = dst_ty, .kind = tmp_kind }, &s); - @memcpy(s_dst_temps[0..dst_temps.len], dst_temps); - - switch (case.each) { - .once => |body| { - for (body) |inst| try s.emit(inst); - s.emitLabel(.@"0:"); - }, - } - assert(s.top == 0); - - for (src_locks) |locks| for (locks) |lock| if (lock) |reg| cg.register_manager.unlockReg(reg); - for (tmp_locks) |locks| for (locks) |lock| if (lock) |reg| cg.register_manager.unlockReg(reg); - for (dst_locks) |locks| for (locks) |lock| if (lock) |reg| cg.register_manager.unlockReg(reg); - caller_preserved: { - const cc = switch (case.clobbers.caller_preserved) { - .none => break :caller_preserved, - .ccc => cg.target.cCallingConvention().?, - .zigcc => .auto, - }; - const err_ret_trace_reg = if (cc == .auto and cg.pt.zcu.comp.config.any_error_tracing) err_ret_trace_reg: { - const param_gpr = abi.getCAbiIntParamRegs(.auto); - break :err_ret_trace_reg param_gpr[param_gpr.len - 1]; - } else .none; - switch (cc) { - else => unreachable, - inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| skip: { - if (reg == err_ret_trace_reg) break :skip; - cg.register_manager.unlockReg(.{ .tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse break :skip }); - }, - } - } - for (tmp_owned, s_tmp_temps) |owned, temp| if (owned) try temp.die(cg); - for (dst_tys, case.dst_temps[0..dst_temps.len], dst_temps) |dst_ty, dst_kind, *dst_temp| try Select.TempSpec.finish(.{ .type = dst_ty, .kind = dst_kind }, dst_temp, cg); - return; - } - } - return error.SelectFailed; -} diff --git a/src/arch/x86_64/Disassembler.zig b/src/arch/x86_64/Disassembler.zig deleted file mode 100644 index 7aeb06f9638fccd2db163c4ad137ee228c455cfb..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/Disassembler.zig +++ /dev/null @@ -1,509 +0,0 @@ -const Disassembler = @This(); - -const std = @import("std"); -const assert = std.debug.assert; -const math = std.math; - -const bits = @import("bits.zig"); -const encoder = @import("encoder.zig"); - -const Encoding = @import("Encoding.zig"); -const Immediate = Instruction.Immediate; -const Instruction = encoder.Instruction; -const LegacyPrefixes = encoder.LegacyPrefixes; -const Memory = Instruction.Memory; -const Register = bits.Register; -const Rex = encoder.Rex; - -pub const Error = error{ - EndOfStream, - /// After the TODO below is solved this will make sense. - ReadFailed, - LegacyPrefixAfterRex, - UnknownOpcode, - Overflow, - Todo, -}; - -// TODO these fields should be replaced by std.Io.Reader - -code: []const u8, -pos: usize = 0, - -pub fn init(code: []const u8) Disassembler { - return .{ .code = code }; -} - -pub fn next(dis: *Disassembler) Error!?Instruction { - const prefixes = dis.parsePrefixes() catch |err| switch (err) { - error.EndOfStream => return null, - else => |e| return e, - }; - - const enc = try dis.parseEncoding(prefixes) orelse return error.UnknownOpcode; - switch (enc.data.op_en) { - .z => return inst(enc, .{}), - .o => { - const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, - }); - }, - .zo => { - const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); - return inst(enc, .{ - .op1 = .{ .reg = enc.data.ops[0].toReg() }, - .op2 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) }, - }); - }, - .oz => { - const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, - .op2 = .{ .reg = enc.data.ops[1].toReg() }, - }); - }, - .oi => { - const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); - const imm = try dis.parseImm(enc.data.ops[1]); - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, - .op2 = .{ .imm = imm }, - }); - }, - .i, .d => { - const imm = try dis.parseImm(enc.data.ops[0]); - return inst(enc, .{ - .op1 = .{ .imm = imm }, - }); - }, - .zi => { - const imm = try dis.parseImm(enc.data.ops[1]); - return inst(enc, .{ - .op1 = .{ .reg = enc.data.ops[0].toReg() }, - .op2 = .{ .imm = imm }, - }); - }, - .ii => { - const imm1 = try dis.parseImm(enc.data.ops[0]); - const imm2 = try dis.parseImm(enc.data.ops[1]); - return inst(enc, .{ - .op1 = .{ .imm = imm1 }, - .op2 = .{ .imm = imm2 }, - }); - }, - .ia => { - const imm = try dis.parseImm(enc.data.ops[0]); - return inst(enc, .{ - .op1 = .{ .imm = imm }, - .op2 = .{ .reg = .eax }, - }); - }, - .m, .mi, .m1, .mc => { - const modrm = try dis.parseModRmByte(); - const act_enc = Encoding.findByOpcode(enc.opcode(), .{ - .legacy = prefixes.legacy, - .rex = prefixes.rex, - }, modrm.op1) orelse return error.UnknownOpcode; - const sib = if (modrm.sib()) try dis.parseSibByte() else null; - - if (modrm.direct()) { - const op2: Instruction.Operand = switch (act_enc.data.op_en) { - .mi => .{ .imm = try dis.parseImm(act_enc.data.ops[1]) }, - .m1 => .{ .imm = Immediate.u(1) }, - .mc => .{ .reg = .cl }, - .m => .none, - else => unreachable, - }; - return inst(act_enc, .{ - .op1 = .{ .reg = parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, act_enc.data.ops[0].regBitSize()) }, - .op2 = op2, - }); - } - - const disp = try dis.parseDisplacement(modrm, sib); - const op2: Instruction.Operand = switch (act_enc.data.op_en) { - .mi => .{ .imm = try dis.parseImm(act_enc.data.ops[1]) }, - .m1 => .{ .imm = Immediate.u(1) }, - .mc => .{ .reg = .cl }, - .m => .none, - else => unreachable, - }; - - if (modrm.rip()) { - return inst(act_enc, .{ - .op1 = .{ .mem = Memory.initRip(Memory.PtrSize.fromBitSize(act_enc.data.ops[0].memBitSize()), disp) }, - .op2 = op2, - }); - } - - const scale_index = if (sib) |info| info.scaleIndex(prefixes.rex) else null; - const base = if (sib) |info| - info.baseReg(modrm, prefixes) - else - parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, 64); - return inst(act_enc, .{ - .op1 = .{ .mem = Memory.initSib(Memory.PtrSize.fromBitSize(act_enc.data.ops[0].memBitSize()), .{ - .base = if (base) |base_reg| .{ .reg = base_reg } else .none, - .scale_index = scale_index, - .disp = disp, - }) }, - .op2 = op2, - }); - }, - .fd => { - const seg = segmentRegister(prefixes.legacy); - const offset = try dis.parseOffset(); - return inst(enc, .{ - .op1 = .{ .reg = enc.data.ops[0].toReg() }, - .op2 = .{ .mem = Memory.initMoffs(seg, offset) }, - }); - }, - .td => { - const seg = segmentRegister(prefixes.legacy); - const offset = try dis.parseOffset(); - return inst(enc, .{ - .op1 = .{ .mem = Memory.initMoffs(seg, offset) }, - .op2 = .{ .reg = enc.data.ops[1].toReg() }, - }); - }, - .mr, .mri, .mrc => { - const modrm = try dis.parseModRmByte(); - const sib = if (modrm.sib()) try dis.parseSibByte() else null; - const src_bit_size = enc.data.ops[1].regBitSize(); - - if (modrm.direct()) { - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, - .op2 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.x, prefixes.rex, src_bit_size) }, - }); - } - - const dst_bit_size = enc.data.ops[0].memBitSize(); - const disp = try dis.parseDisplacement(modrm, sib); - const op3: Instruction.Operand = switch (enc.data.op_en) { - .mri => .{ .imm = try dis.parseImm(enc.data.ops[2]) }, - .mrc => .{ .reg = .cl }, - .mr => .none, - else => unreachable, - }; - - if (modrm.rip()) { - return inst(enc, .{ - .op1 = .{ .mem = Memory.initRip(Memory.PtrSize.fromBitSize(dst_bit_size), disp) }, - .op2 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, src_bit_size) }, - .op3 = op3, - }); - } - - const scale_index = if (sib) |info| info.scaleIndex(prefixes.rex) else null; - const base = if (sib) |info| - info.baseReg(modrm, prefixes) - else - parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, 64); - return inst(enc, .{ - .op1 = .{ .mem = Memory.initSib(Memory.PtrSize.fromBitSize(dst_bit_size), .{ - .base = if (base) |base_reg| .{ .reg = base_reg } else .none, - .scale_index = scale_index, - .disp = disp, - }) }, - .op2 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, src_bit_size) }, - .op3 = op3, - }); - }, - .rm, .rmi => { - const modrm = try dis.parseModRmByte(); - const sib = if (modrm.sib()) try dis.parseSibByte() else null; - const dst_bit_size = enc.data.ops[0].regBitSize(); - - if (modrm.direct()) { - const op3: Instruction.Operand = switch (enc.data.op_en) { - .rm => .none, - .rmi => .{ .imm = try dis.parseImm(enc.data.ops[2]) }, - else => unreachable, - }; - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.x, prefixes.rex, dst_bit_size) }, - .op2 = .{ .reg = parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) }, - .op3 = op3, - }); - } - - const src_bit_size = if (enc.data.ops[1] == .m) dst_bit_size else enc.data.ops[1].memBitSize(); - const disp = try dis.parseDisplacement(modrm, sib); - const op3: Instruction.Operand = switch (enc.data.op_en) { - .rmi => .{ .imm = try dis.parseImm(enc.data.ops[2]) }, - .rm => .none, - else => unreachable, - }; - - if (modrm.rip()) { - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, dst_bit_size) }, - .op2 = .{ .mem = Memory.initRip(Memory.PtrSize.fromBitSize(src_bit_size), disp) }, - .op3 = op3, - }); - } - - const scale_index = if (sib) |info| info.scaleIndex(prefixes.rex) else null; - const base = if (sib) |info| - info.baseReg(modrm, prefixes) - else - parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, 64); - return inst(enc, .{ - .op1 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, dst_bit_size) }, - .op2 = .{ .mem = Memory.initSib(Memory.PtrSize.fromBitSize(src_bit_size), .{ - .base = if (base) |base_reg| .{ .reg = base_reg } else .none, - .scale_index = scale_index, - .disp = disp, - }) }, - .op3 = op3, - }); - }, - .rm0, .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable, // TODO - } -} - -fn inst(encoding: Encoding, args: struct { - prefix: Instruction.Prefix = .none, - op1: Instruction.Operand = .none, - op2: Instruction.Operand = .none, - op3: Instruction.Operand = .none, - op4: Instruction.Operand = .none, -}) Instruction { - return .{ .encoding = encoding, .prefix = args.prefix, .ops = .{ - args.op1, - args.op2, - args.op3, - args.op4, - } }; -} - -const Prefixes = struct { - legacy: LegacyPrefixes = .{}, - rex: Rex = .{}, - // TODO add support for VEX prefix -}; - -fn parsePrefixes(dis: *Disassembler) !Prefixes { - const rex_prefix_mask: u4 = 0b0100; - var reader: std.Io.Reader = .fixed(dis.code[dis.pos..]); - - var res: Prefixes = .{}; - - while (true) { - const next_byte = try reader.takeByte(); - dis.pos += 1; - - switch (next_byte) { - 0xf0, 0xf2, 0xf3, 0x2e, 0x36, 0x26, 0x64, 0x65, 0x3e, 0x66, 0x67 => { - // Legacy prefix - if (res.rex.present) return error.LegacyPrefixAfterRex; - switch (next_byte) { - 0xf0 => res.legacy.prefix_f0 = true, - 0xf2 => res.legacy.prefix_f2 = true, - 0xf3 => res.legacy.prefix_f3 = true, - 0x2e => res.legacy.prefix_2e = true, - 0x36 => res.legacy.prefix_36 = true, - 0x26 => res.legacy.prefix_26 = true, - 0x64 => res.legacy.prefix_64 = true, - 0x65 => res.legacy.prefix_65 = true, - 0x3e => res.legacy.prefix_3e = true, - 0x66 => res.legacy.prefix_66 = true, - 0x67 => res.legacy.prefix_67 = true, - else => unreachable, - } - }, - else => { - if (rex_prefix_mask == @as(u4, @truncate(next_byte >> 4))) { - // REX prefix - res.rex.w = next_byte & 0b1000 != 0; - res.rex.r = next_byte & 0b100 != 0; - res.rex.x = next_byte & 0b10 != 0; - res.rex.b = next_byte & 0b1 != 0; - res.rex.present = true; - continue; - } - - // TODO VEX prefix - - dis.pos -= 1; - break; - }, - } - } - - return res; -} - -fn parseEncoding(dis: *Disassembler, prefixes: Prefixes) !?Encoding { - const o_mask: u8 = 0b1111_1000; - - var opcode: [3]u8 = .{ 0, 0, 0 }; - var reader: std.Io.Reader = .fixed(dis.code[dis.pos..]); - - comptime var opc_count = 0; - inline while (opc_count < 3) : (opc_count += 1) { - const byte = try reader.takeByte(); - opcode[opc_count] = byte; - dis.pos += 1; - - if (byte == 0x0f) { - // Multi-byte opcode - } else if (opc_count > 0) { - // Multi-byte opcode - if (Encoding.findByOpcode(opcode[0 .. opc_count + 1], .{ - .legacy = prefixes.legacy, - .rex = prefixes.rex, - }, null)) |mnemonic| { - return mnemonic; - } - } else { - // Single-byte opcode - if (Encoding.findByOpcode(opcode[0..1], .{ - .legacy = prefixes.legacy, - .rex = prefixes.rex, - }, null)) |mnemonic| { - return mnemonic; - } else { - // Try O* encoding - return Encoding.findByOpcode(&.{opcode[0] & o_mask}, .{ - .legacy = prefixes.legacy, - .rex = prefixes.rex, - }, null); - } - } - } - return null; -} - -fn parseGpRegister(low_enc: u3, is_extended: bool, rex: Rex, bit_size: u64) Register { - const reg_id: u4 = @as(u4, @intCast(@intFromBool(is_extended))) << 3 | low_enc; - const reg = @as(Register, @enumFromInt(reg_id)).toBitSize(bit_size); - return switch (reg) { - .spl => if (rex.present or rex.isSet()) .spl else .ah, - .dil => if (rex.present or rex.isSet()) .dil else .bh, - .bpl => if (rex.present or rex.isSet()) .bpl else .ch, - .sil => if (rex.present or rex.isSet()) .sil else .dh, - else => reg, - }; -} - -fn parseImm(dis: *Disassembler, kind: Encoding.Op) !Immediate { - var reader: std.Io.Reader = .fixed(dis.code); - reader.seek = dis.pos; - defer dis.pos = reader.seek; - - const imm = switch (kind) { - .imm8s, .rel8 => Immediate.s(try reader.takeInt(i8, .little)), - .imm16s, .rel16 => Immediate.s(try reader.takeInt(i16, .little)), - .imm32s, .rel32 => Immediate.s(try reader.takeInt(i32, .little)), - .imm8 => Immediate.u(try reader.takeInt(u8, .little)), - .imm16 => Immediate.u(try reader.takeInt(u16, .little)), - .imm32 => Immediate.u(try reader.takeInt(u32, .little)), - .imm64 => Immediate.u(try reader.takeInt(u64, .little)), - else => unreachable, - }; - return imm; -} - -fn parseOffset(dis: *Disassembler) !u64 { - var reader: std.Io.Reader = .fixed(dis.code); - reader.seek = dis.pos; - defer dis.pos = reader.seek; - - return reader.takeInt(u64, .little); -} - -const ModRm = packed struct { - mod: u2, - op1: u3, - op2: u3, - - inline fn direct(self: ModRm) bool { - return self.mod == 0b11; - } - - inline fn rip(self: ModRm) bool { - return self.mod == 0 and self.op2 == 0b101; - } - - inline fn sib(self: ModRm) bool { - return !self.direct() and self.op2 == 0b100; - } -}; - -fn parseModRmByte(dis: *Disassembler) !ModRm { - if (dis.code[dis.pos..].len == 0) return error.EndOfStream; - const modrm_byte = dis.code[dis.pos]; - dis.pos += 1; - const mod: u2 = @as(u2, @truncate(modrm_byte >> 6)); - const op1: u3 = @as(u3, @truncate(modrm_byte >> 3)); - const op2: u3 = @as(u3, @truncate(modrm_byte)); - return ModRm{ .mod = mod, .op1 = op1, .op2 = op2 }; -} - -fn segmentRegister(prefixes: LegacyPrefixes) Register { - if (prefixes.prefix_2e) return .cs; - if (prefixes.prefix_36) return .ss; - if (prefixes.prefix_26) return .es; - if (prefixes.prefix_64) return .fs; - if (prefixes.prefix_65) return .gs; - return .ds; -} - -const Sib = packed struct { - scale: u2, - index: u3, - base: u3, - - fn scaleIndex(self: Sib, rex: Rex) ?Memory.ScaleIndex { - if (self.index == 0b100 and !rex.x) return null; - return .{ - .scale = @as(u4, 1) << self.scale, - .index = parseGpRegister(self.index, rex.x, rex, 64), - }; - } - - fn baseReg(self: Sib, modrm: ModRm, prefixes: Prefixes) ?Register { - if (self.base == 0b101 and modrm.mod == 0) { - if (self.scaleIndex(prefixes.rex)) |_| return null; - return segmentRegister(prefixes.legacy); - } - return parseGpRegister(self.base, prefixes.rex.b, prefixes.rex, 64); - } -}; - -fn parseSibByte(dis: *Disassembler) !Sib { - if (dis.code[dis.pos..].len == 0) return error.EndOfStream; - const sib_byte = dis.code[dis.pos]; - dis.pos += 1; - const scale: u2 = @as(u2, @truncate(sib_byte >> 6)); - const index: u3 = @as(u3, @truncate(sib_byte >> 3)); - const base: u3 = @as(u3, @truncate(sib_byte)); - return Sib{ .scale = scale, .index = index, .base = base }; -} - -fn parseDisplacement(dis: *Disassembler, modrm: ModRm, sib: ?Sib) !i32 { - var reader: std.Io.Reader = .fixed(dis.code); - reader.seek = dis.pos; - defer dis.pos = reader.seek; - - const disp = disp: { - if (sib) |info| { - if (info.base == 0b101 and modrm.mod == 0) { - break :disp try reader.takeInt(i32, .little); - } - } - if (modrm.rip()) { - break :disp try reader.takeInt(i32, .little); - } - break :disp switch (modrm.mod) { - 0b00 => 0, - 0b01 => try reader.takeInt(i8, .little), - 0b10 => try reader.takeInt(i32, .little), - 0b11 => unreachable, - }; - }; - return disp; -} diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig deleted file mode 100644 index c2b38d8e6d45b3439d2f8bb6cf9e518ac17784cc..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/Emit.zig +++ /dev/null @@ -1,982 +0,0 @@ -//! This file contains the functionality for emitting x86_64 MIR as machine code - -lower: Lower, -bin_file: *link.File, -pt: Zcu.PerThread, -pic: bool, -atom_index: u32, -debug_output: link.File.DebugInfoOutput, -w: *std.Io.Writer, - -prev_di_loc: Loc, -/// Relative to the beginning of `code`. -prev_di_pc: usize, - -code_offset_mapping: std.ArrayListUnmanaged(u32), -relocs: std.ArrayListUnmanaged(Reloc), -table_relocs: std.ArrayListUnmanaged(TableReloc), - -pub const Error = Lower.Error || error{ - EmitFail, - NotFile, -} || std.posix.MMapError || std.posix.MRemapError || link.File.UpdateDebugInfoError; - -pub fn emitMir(emit: *Emit) Error!void { - const comp = emit.bin_file.comp; - const gpa = comp.gpa; - try emit.code_offset_mapping.resize(gpa, emit.lower.mir.instructions.len); - emit.relocs.clearRetainingCapacity(); - emit.table_relocs.clearRetainingCapacity(); - var local_index: usize = 0; - for (0..emit.lower.mir.instructions.len) |mir_i| { - const mir_index: Mir.Inst.Index = @intCast(mir_i); - emit.code_offset_mapping.items[mir_index] = @intCast(emit.w.end); - const lowered = try emit.lower.lowerMir(mir_index); - var lowered_relocs = lowered.relocs; - lowered_inst: for (lowered.insts, 0..) |lowered_inst, lowered_index| { - if (lowered_inst.prefix == .directive) { - const start_offset: u32 = @intCast(emit.w.end); - switch (emit.debug_output) { - .dwarf => |dwarf| switch (lowered_inst.encoding.mnemonic) { - .@".cfi_def_cfa" => try dwarf.genDebugFrame(start_offset, .{ .def_cfa = .{ - .reg = lowered_inst.ops[0].reg.dwarfNum(), - .off = lowered_inst.ops[1].imm.signed, - } }), - .@".cfi_def_cfa_register" => try dwarf.genDebugFrame(start_offset, .{ - .def_cfa_register = lowered_inst.ops[0].reg.dwarfNum(), - }), - .@".cfi_def_cfa_offset" => try dwarf.genDebugFrame(start_offset, .{ - .def_cfa_offset = lowered_inst.ops[0].imm.signed, - }), - .@".cfi_adjust_cfa_offset" => try dwarf.genDebugFrame(start_offset, .{ - .adjust_cfa_offset = lowered_inst.ops[0].imm.signed, - }), - .@".cfi_offset" => try dwarf.genDebugFrame(start_offset, .{ .offset = .{ - .reg = lowered_inst.ops[0].reg.dwarfNum(), - .off = lowered_inst.ops[1].imm.signed, - } }), - .@".cfi_val_offset" => try dwarf.genDebugFrame(start_offset, .{ .val_offset = .{ - .reg = lowered_inst.ops[0].reg.dwarfNum(), - .off = lowered_inst.ops[1].imm.signed, - } }), - .@".cfi_rel_offset" => try dwarf.genDebugFrame(start_offset, .{ .rel_offset = .{ - .reg = lowered_inst.ops[0].reg.dwarfNum(), - .off = lowered_inst.ops[1].imm.signed, - } }), - .@".cfi_register" => try dwarf.genDebugFrame(start_offset, .{ .register = .{ - lowered_inst.ops[0].reg.dwarfNum(), - lowered_inst.ops[1].reg.dwarfNum(), - } }), - .@".cfi_restore" => try dwarf.genDebugFrame(start_offset, .{ - .restore = lowered_inst.ops[0].reg.dwarfNum(), - }), - .@".cfi_undefined" => try dwarf.genDebugFrame(start_offset, .{ - .undefined = lowered_inst.ops[0].reg.dwarfNum(), - }), - .@".cfi_same_value" => try dwarf.genDebugFrame(start_offset, .{ - .same_value = lowered_inst.ops[0].reg.dwarfNum(), - }), - .@".cfi_remember_state" => try dwarf.genDebugFrame(start_offset, .remember_state), - .@".cfi_restore_state" => try dwarf.genDebugFrame(start_offset, .restore_state), - .@".cfi_escape" => try dwarf.genDebugFrame(start_offset, .{ - .escape = lowered_inst.ops[0].bytes, - }), - else => unreachable, - }, - .none => {}, - } - continue; - } - var reloc_info_buf: [2]RelocInfo = undefined; - var reloc_info_index: usize = 0; - while (lowered_relocs.len > 0 and - lowered_relocs[0].lowered_inst_index == lowered_index) : ({ - lowered_relocs = lowered_relocs[1..]; - reloc_info_index += 1; - }) reloc_info_buf[reloc_info_index] = .{ - .op_index = lowered_relocs[0].op_index, - .off = lowered_relocs[0].off, - .target = target: switch (lowered_relocs[0].target) { - .inst => |inst| .{ .index = inst, .is_extern = false, .type = .inst }, - .table => .{ .index = undefined, .is_extern = false, .type = .table }, - .nav => |nav| { - const sym_index = switch (try codegen.genNavRef( - emit.bin_file, - emit.pt, - emit.lower.src_loc, - nav, - emit.lower.target, - )) { - .sym_index => |sym_index| sym_index, - .fail => |em| { - assert(emit.lower.err_msg == null); - emit.lower.err_msg = em; - return error.EmitFail; - }, - }; - const ip = &emit.pt.zcu.intern_pool; - break :target switch (ip.getNav(nav).status) { - .unresolved => unreachable, - .type_resolved => |type_resolved| .{ - .index = sym_index, - .is_extern = false, - .type = if (type_resolved.is_threadlocal and comp.config.any_non_single_threaded) .tlv else .symbol, - }, - .fully_resolved => |fully_resolved| switch (ip.indexToKey(fully_resolved.val)) { - .@"extern" => |@"extern"| .{ - .index = sym_index, - .is_extern = switch (@"extern".visibility) { - .default => true, - .hidden, .protected => false, - }, - .type = if (@"extern".is_threadlocal and comp.config.any_non_single_threaded) .tlv else .symbol, - .force_pcrel_direct = switch (@"extern".relocation) { - .any => false, - .pcrel => true, - }, - }, - .variable => |variable| .{ - .index = sym_index, - .is_extern = false, - .type = if (variable.is_threadlocal and comp.config.any_non_single_threaded) .tlv else .symbol, - }, - else => .{ .index = sym_index, .is_extern = false, .type = .symbol }, - }, - }; - }, - .uav => |uav| .{ - .index = switch (try emit.bin_file.lowerUav( - emit.pt, - uav.val, - Type.fromInterned(uav.orig_ty).ptrAlignment(emit.pt.zcu), - emit.lower.src_loc, - )) { - .sym_index => |sym_index| sym_index, - .fail => |em| { - assert(emit.lower.err_msg == null); - emit.lower.err_msg = em; - return error.EmitFail; - }, - }, - .is_extern = false, - .type = .symbol, - }, - .lazy_sym => |lazy_sym| .{ - .index = if (emit.bin_file.cast(.elf)) |elf_file| - elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, emit.pt, lazy_sym) catch |err| - return emit.fail("{s} creating lazy symbol", .{@errorName(err)}) - else if (emit.bin_file.cast(.elf2)) |elf| - @intFromEnum(try elf.lazySymbol(lazy_sym)) - else if (emit.bin_file.cast(.macho)) |macho_file| - macho_file.getZigObject().?.getOrCreateMetadataForLazySymbol(macho_file, emit.pt, lazy_sym) catch |err| - return emit.fail("{s} creating lazy symbol", .{@errorName(err)}) - else if (emit.bin_file.cast(.coff)) |coff_file| - if (coff_file.getOrCreateAtomForLazySymbol(emit.pt, lazy_sym)) |atom| - coff_file.getAtom(atom).getSymbolIndex().? - else |err| - return emit.fail("{s} creating lazy symbol", .{@errorName(err)}) - else - return emit.fail("lazy symbols unimplemented for {s}", .{@tagName(emit.bin_file.tag)}), - .is_extern = false, - .type = .symbol, - }, - .extern_func => |extern_func| .{ - .index = if (emit.bin_file.cast(.elf)) |elf_file| - try elf_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, null) - else if (emit.bin_file.cast(.elf2)) |elf| @intFromEnum(try elf.globalSymbol(.{ - .name = extern_func.toSlice(&emit.lower.mir).?, - .type = .FUNC, - })) else if (emit.bin_file.cast(.macho)) |macho_file| - try macho_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, null) - else if (emit.bin_file.cast(.coff)) |coff_file| - try coff_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, "compiler_rt") - else - return emit.fail("external symbol unimplemented for {s}", .{@tagName(emit.bin_file.tag)}), - .is_extern = true, - .type = .symbol, - }, - }, - }; - const reloc_info = reloc_info_buf[0..reloc_info_index]; - for (reloc_info) |*reloc| switch (reloc.target.type) { - .inst, .table => {}, - .symbol => { - switch (lowered_inst.encoding.mnemonic) { - .call => { - reloc.target.type = .branch; - if (emit.bin_file.cast(.coff)) |_| try emit.encodeInst(try .new(.none, .call, &.{ - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info) else try emit.encodeInst(lowered_inst, reloc_info); - continue :lowered_inst; - }, - else => {}, - } - if (emit.bin_file.cast(.elf) != null or emit.bin_file.cast(.elf2) != null) { - if (!emit.pic) switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .imm = .s(0) }, - }, emit.lower.target), reloc_info), - .mov => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ - .base = .{ .reg = .ds }, - }) }, - }, emit.lower.target), reloc_info), - else => unreachable, - } else if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info), - .mov => { - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info); - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{ - .reg = lowered_inst.ops[0].reg.to64(), - } }) }, - }, emit.lower.target), &.{}); - }, - else => unreachable, - } else switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .lea, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.none, 0) }, - }, emit.lower.target), reloc_info), - .mov => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, 0) }, - }, emit.lower.target), reloc_info), - else => unreachable, - } - } else if (emit.bin_file.cast(.macho)) |_| { - if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info), - .mov => { - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info); - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{ - .reg = lowered_inst.ops[0].reg.to64(), - } }) }, - }, emit.lower.target), &.{}); - }, - else => unreachable, - } else switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .lea, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.none, 0) }, - }, emit.lower.target), reloc_info), - .mov => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, 0) }, - }, emit.lower.target), reloc_info), - else => unreachable, - } - } else if (emit.bin_file.cast(.coff)) |_| { - if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info), - .mov => { - const dst_reg = lowered_inst.ops[0].reg.to64(); - try emit.encodeInst(try .new(.none, .mov, &.{ - .{ .reg = dst_reg }, - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info); - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{ - .reg = dst_reg, - } }) }, - }, emit.lower.target), &.{}); - }, - else => unreachable, - } else switch (lowered_inst.encoding.mnemonic) { - .lea => try emit.encodeInst(try .new(.none, .lea, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(.none, 0) }, - }, emit.lower.target), reloc_info), - .mov => try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initRip(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, 0) }, - }, emit.lower.target), reloc_info), - else => unreachable, - } - } else return emit.fail("TODO implement relocs for {s}", .{ - @tagName(emit.bin_file.tag), - }); - continue :lowered_inst; - }, - .branch, .tls => unreachable, - .tlv => { - if (emit.bin_file.cast(.elf) != null or emit.bin_file.cast(.elf2) != null) { - // TODO handle extern TLS vars, i.e., emit GD model - if (emit.pic) switch (lowered_inst.encoding.mnemonic) { - .lea, .mov => { - // Here, we currently assume local dynamic TLS vars, and so - // we emit LD model. - try emit.encodeInst(try .new(.none, .lea, &.{ - .{ .reg = .rdi }, - .{ .mem = .initRip(.none, 0) }, - }, emit.lower.target), &.{.{ - .op_index = 1, - .target = .{ - .index = reloc.target.index, - .is_extern = false, - .type = .tls, - }, - }}); - try emit.encodeInst(try .new(.none, .call, &.{ - .{ .imm = .s(0) }, - }, emit.lower.target), &.{.{ - .op_index = 0, - .target = .{ - .index = if (emit.bin_file.cast(.elf)) |elf_file| - try elf_file.getGlobalSymbol("__tls_get_addr", null) - else if (emit.bin_file.cast(.elf2)) |elf| @intFromEnum(try elf.globalSymbol(.{ - .name = "__tls_get_addr", - .type = .FUNC, - })) else unreachable, - .is_extern = true, - .type = .branch, - }, - }}); - try emit.encodeInst(try .new(.none, lowered_inst.encoding.mnemonic, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(.none, .{ - .base = .{ .reg = .rax }, - .disp = std.math.minInt(i32), - }) }, - }, emit.lower.target), reloc_info); - }, - else => unreachable, - } else switch (lowered_inst.encoding.mnemonic) { - .lea, .mov => { - // Since we are linking statically, we emit LE model directly. - try emit.encodeInst(try .new(.none, .mov, &.{ - .{ .reg = .rax }, - .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .fs } }) }, - }, emit.lower.target), &.{}); - try emit.encodeInst(try .new(.none, lowered_inst.encoding.mnemonic, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(.none, .{ - .base = .{ .reg = .rax }, - .disp = std.math.minInt(i32), - }) }, - }, emit.lower.target), reloc_info); - }, - else => unreachable, - } - } else if (emit.bin_file.cast(.macho)) |_| switch (lowered_inst.encoding.mnemonic) { - .lea => { - try emit.encodeInst(try .new(.none, .mov, &.{ - .{ .reg = .rdi }, - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info); - try emit.encodeInst(try .new(.none, .call, &.{ - .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .rdi } }) }, - }, emit.lower.target), &.{}); - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .reg = .rax }, - }, emit.lower.target), &.{}); - }, - .mov => { - try emit.encodeInst(try .new(.none, .mov, &.{ - .{ .reg = .rdi }, - .{ .mem = .initRip(.ptr, 0) }, - }, emit.lower.target), reloc_info); - try emit.encodeInst(try .new(.none, .call, &.{ - .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .rdi } }) }, - }, emit.lower.target), &.{}); - try emit.encodeInst(try .new(.none, .mov, &.{ - lowered_inst.ops[0], - .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .rax } }) }, - }, emit.lower.target), &.{}); - }, - else => unreachable, - } else return emit.fail("TODO implement relocs for {s}", .{ - @tagName(emit.bin_file.tag), - }); - continue :lowered_inst; - }, - }; - try emit.encodeInst(lowered_inst, reloc_info); - } - assert(lowered_relocs.len == 0); - - if (lowered.insts.len == 0) { - const mir_inst = emit.lower.mir.instructions.get(mir_index); - switch (mir_inst.tag) { - else => unreachable, - .pseudo => switch (mir_inst.ops) { - else => unreachable, - .pseudo_dbg_prologue_end_none => switch (emit.debug_output) { - .dwarf => |dwarf| try dwarf.setPrologueEnd(), - .none => {}, - }, - .pseudo_dbg_line_stmt_line_column => try emit.dbgAdvancePCAndLine(.{ - .line = mir_inst.data.line_column.line, - .column = mir_inst.data.line_column.column, - .is_stmt = true, - }), - .pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(.{ - .line = mir_inst.data.line_column.line, - .column = mir_inst.data.line_column.column, - .is_stmt = false, - }), - .pseudo_dbg_epilogue_begin_none => switch (emit.debug_output) { - .dwarf => |dwarf| { - try dwarf.setEpilogueBegin(); - log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{ - emit.prev_di_loc.line, emit.prev_di_loc.column, - }); - try emit.dbgAdvancePCAndLine(emit.prev_di_loc); - }, - .none => {}, - }, - .pseudo_dbg_enter_block_none => switch (emit.debug_output) { - .dwarf => |dwarf| { - log.debug("mirDbgEnterBlock (line={d}, col={d})", .{ - emit.prev_di_loc.line, emit.prev_di_loc.column, - }); - try dwarf.enterBlock(emit.w.end); - }, - .none => {}, - }, - .pseudo_dbg_leave_block_none => switch (emit.debug_output) { - .dwarf => |dwarf| { - log.debug("mirDbgLeaveBlock (line={d}, col={d})", .{ - emit.prev_di_loc.line, emit.prev_di_loc.column, - }); - try dwarf.leaveBlock(emit.w.end); - }, - .none => {}, - }, - .pseudo_dbg_enter_inline_func => switch (emit.debug_output) { - .dwarf => |dwarf| { - log.debug("mirDbgEnterInline (line={d}, col={d})", .{ - emit.prev_di_loc.line, emit.prev_di_loc.column, - }); - try dwarf.enterInlineFunc(mir_inst.data.ip_index, emit.w.end, emit.prev_di_loc.line, emit.prev_di_loc.column); - }, - .none => {}, - }, - .pseudo_dbg_leave_inline_func => switch (emit.debug_output) { - .dwarf => |dwarf| { - log.debug("mirDbgLeaveInline (line={d}, col={d})", .{ - emit.prev_di_loc.line, emit.prev_di_loc.column, - }); - try dwarf.leaveInlineFunc(mir_inst.data.ip_index, emit.w.end); - }, - .none => {}, - }, - .pseudo_dbg_arg_none, - .pseudo_dbg_arg_i_s, - .pseudo_dbg_arg_i_u, - .pseudo_dbg_arg_i_64, - .pseudo_dbg_arg_ro, - .pseudo_dbg_arg_fa, - .pseudo_dbg_arg_m, - .pseudo_dbg_var_none, - .pseudo_dbg_var_i_s, - .pseudo_dbg_var_i_u, - .pseudo_dbg_var_i_64, - .pseudo_dbg_var_ro, - .pseudo_dbg_var_fa, - .pseudo_dbg_var_m, - => switch (emit.debug_output) { - .dwarf => |dwarf| { - var loc_buf: [2]link.File.Dwarf.Loc = undefined; - const loc: link.File.Dwarf.Loc = loc: switch (mir_inst.ops) { - else => unreachable, - .pseudo_dbg_arg_none, .pseudo_dbg_var_none => .empty, - .pseudo_dbg_arg_i_s, - .pseudo_dbg_arg_i_u, - .pseudo_dbg_var_i_s, - .pseudo_dbg_var_i_u, - => .{ .stack_value = stack_value: { - loc_buf[0] = switch (emit.lower.imm(mir_inst.ops, mir_inst.data.i.i)) { - .signed => |s| .{ .consts = s }, - .unsigned => |u| .{ .constu = u }, - }; - break :stack_value &loc_buf[0]; - } }, - .pseudo_dbg_arg_i_64, .pseudo_dbg_var_i_64 => .{ .stack_value = stack_value: { - loc_buf[0] = .{ .constu = mir_inst.data.i64 }; - break :stack_value &loc_buf[0]; - } }, - .pseudo_dbg_arg_fa, .pseudo_dbg_var_fa => { - const reg_off = emit.lower.mir.resolveFrameAddr(mir_inst.data.fa); - break :loc .{ .plus = .{ - reg: { - loc_buf[0] = .{ .breg = reg_off.reg.dwarfNum() }; - break :reg &loc_buf[0]; - }, - off: { - loc_buf[1] = .{ .consts = reg_off.off }; - break :off &loc_buf[1]; - }, - } }; - }, - .pseudo_dbg_arg_m, .pseudo_dbg_var_m => { - const mem = emit.lower.mir.resolveMemoryExtra(mir_inst.data.x.payload).decode(); - break :loc .{ .plus = .{ - base: { - loc_buf[0] = switch (mem.base()) { - .none => .{ .constu = 0 }, - .reg => |reg| .{ .breg = reg.dwarfNum() }, - .frame, .table, .rip_inst => unreachable, - .nav => |nav| .{ .addr_reloc = switch (codegen.genNavRef( - emit.bin_file, - emit.pt, - emit.lower.src_loc, - nav, - emit.lower.target, - ) catch |err| switch (err) { - error.CodegenFail, - => return emit.fail("unable to codegen: {s}", .{@errorName(err)}), - else => |e| return e, - }) { - .sym_index => |sym_index| sym_index, - .fail => |em| { - assert(emit.lower.err_msg == null); - emit.lower.err_msg = em; - return error.EmitFail; - }, - } }, - .uav => |uav| .{ .addr_reloc = switch (try emit.bin_file.lowerUav( - emit.pt, - uav.val, - Type.fromInterned(uav.orig_ty).ptrAlignment(emit.pt.zcu), - emit.lower.src_loc, - )) { - .sym_index => |sym_index| sym_index, - .fail => |em| { - assert(emit.lower.err_msg == null); - emit.lower.err_msg = em; - return error.EmitFail; - }, - } }, - .lazy_sym, .extern_func => unreachable, - }; - break :base &loc_buf[0]; - }, - disp: { - loc_buf[1] = switch (mem.disp()) { - .signed => |s| .{ .consts = s }, - .unsigned => |u| .{ .constu = u }, - }; - break :disp &loc_buf[1]; - }, - } }; - }, - }; - - const local = &emit.lower.mir.locals[local_index]; - local_index += 1; - try dwarf.genLocalVarDebugInfo( - switch (mir_inst.ops) { - else => unreachable, - .pseudo_dbg_arg_none, - .pseudo_dbg_arg_i_s, - .pseudo_dbg_arg_i_u, - .pseudo_dbg_arg_i_64, - .pseudo_dbg_arg_ro, - .pseudo_dbg_arg_fa, - .pseudo_dbg_arg_m, - .pseudo_dbg_arg_val, - => .arg, - .pseudo_dbg_var_none, - .pseudo_dbg_var_i_s, - .pseudo_dbg_var_i_u, - .pseudo_dbg_var_i_64, - .pseudo_dbg_var_ro, - .pseudo_dbg_var_fa, - .pseudo_dbg_var_m, - .pseudo_dbg_var_val, - => .local_var, - }, - local.name.toSlice(&emit.lower.mir), - .fromInterned(local.type), - loc, - ); - }, - .none => local_index += 1, - }, - .pseudo_dbg_arg_val, .pseudo_dbg_var_val => switch (emit.debug_output) { - .dwarf => |dwarf| { - const local = &emit.lower.mir.locals[local_index]; - local_index += 1; - try dwarf.genLocalConstDebugInfo( - emit.lower.src_loc, - switch (mir_inst.ops) { - else => unreachable, - .pseudo_dbg_arg_val => .comptime_arg, - .pseudo_dbg_var_val => .local_const, - }, - local.name.toSlice(&emit.lower.mir), - .fromInterned(mir_inst.data.ip_index), - ); - }, - .none => local_index += 1, - }, - .pseudo_dbg_var_args_none => switch (emit.debug_output) { - .dwarf => |dwarf| try dwarf.genVarArgsDebugInfo(), - .none => {}, - }, - .pseudo_dead_none => {}, - }, - } - } - } - for (emit.relocs.items) |reloc| { - const target = emit.code_offset_mapping.items[reloc.target]; - const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.inst_offset + reloc.inst_length)) + reloc.target_offset; - const inst_bytes = emit.w.buffered()[reloc.inst_offset..][0..reloc.inst_length]; - switch (reloc.source_length) { - else => unreachable, - inline 1, 4 => |source_length| std.mem.writeInt( - @Type(.{ .int = .{ .signedness = .signed, .bits = @as(u16, 8) * source_length } }), - inst_bytes[reloc.source_offset..][0..source_length], - @intCast(disp), - .little, - ), - } - } - if (emit.lower.mir.table.len > 0) { - const ptr_size = @divExact(emit.lower.target.ptrBitWidth(), 8); - var table_offset = std.mem.alignForward(u32, @intCast(emit.w.end), ptr_size); - if (emit.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const atom = zo.symbol(emit.atom_index).atom(elf_file).?; - - for (emit.table_relocs.items) |table_reloc| try atom.addReloc(gpa, .{ - .r_offset = table_reloc.source_offset, - .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"32"), - .r_addend = @as(i64, table_offset) + table_reloc.target_offset, - }, zo); - for (emit.lower.mir.table) |entry| { - try atom.addReloc(gpa, .{ - .r_offset = table_offset, - .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"64"), - .r_addend = emit.code_offset_mapping.items[entry], - }, zo); - table_offset += ptr_size; - } - try emit.w.splatByteAll(0, table_offset - emit.w.end); - } else if (emit.bin_file.cast(.elf2)) |elf| { - for (emit.table_relocs.items) |table_reloc| try elf.addReloc( - @enumFromInt(emit.atom_index), - table_reloc.source_offset, - @enumFromInt(emit.atom_index), - @as(i64, table_offset) + table_reloc.target_offset, - .{ .x86_64 = .@"32" }, - ); - for (emit.lower.mir.table) |entry| { - try elf.addReloc( - @enumFromInt(emit.atom_index), - table_offset, - @enumFromInt(emit.atom_index), - emit.code_offset_mapping.items[entry], - .{ .x86_64 = .@"64" }, - ); - table_offset += ptr_size; - } - try emit.w.splatByteAll(0, table_offset - emit.w.end); - } else unreachable; - } -} - -pub fn deinit(emit: *Emit) void { - const gpa = emit.bin_file.comp.gpa; - emit.code_offset_mapping.deinit(gpa); - emit.relocs.deinit(gpa); - emit.table_relocs.deinit(gpa); - emit.* = undefined; -} - -const RelocInfo = struct { - op_index: Lower.InstOpIndex, - off: i32 = 0, - target: Target, - - const Target = struct { - index: u32, - is_extern: bool, - type: Target.Type, - force_pcrel_direct: bool = false, - - const Type = enum { inst, table, symbol, branch, tls, tlv }; - }; -}; - -fn encodeInst(emit: *Emit, lowered_inst: Instruction, reloc_info: []const RelocInfo) Error!void { - const comp = emit.bin_file.comp; - const gpa = comp.gpa; - const start_offset: u32 = @intCast(emit.w.end); - lowered_inst.encode(emit.w, .{}) catch |err| switch (err) { - error.WriteFailed => return error.OutOfMemory, - else => |e| return e, - }; - const end_offset: u32 = @intCast(emit.w.end); - for (reloc_info) |reloc| switch (reloc.target.type) { - .inst => { - const inst_length: u4 = @intCast(end_offset - start_offset); - const reloc_offset, const reloc_length = reloc_offset_length: { - var reloc_offset = inst_length; - var op_index: usize = lowered_inst.ops.len; - while (true) { - op_index -= 1; - const op = lowered_inst.encoding.data.ops[op_index]; - if (op == .none) continue; - const is_mem = op.isMemory(); - const enc_length: u4 = if (is_mem) switch (lowered_inst.ops[op_index].mem.sib.base) { - .rip_inst => 4, - else => unreachable, - } else @intCast(std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable); - reloc_offset -= enc_length; - if (op_index == reloc.op_index) break :reloc_offset_length .{ reloc_offset, enc_length }; - assert(!is_mem); - } - }; - try emit.relocs.append(emit.lower.allocator, .{ - .inst_offset = start_offset, - .inst_length = inst_length, - .source_offset = reloc_offset, - .source_length = reloc_length, - .target = reloc.target.index, - .target_offset = reloc.off, - }); - }, - .table => try emit.table_relocs.append(emit.lower.allocator, .{ - .source_offset = end_offset - 4, - .target_offset = reloc.off, - }), - .symbol => if (emit.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const atom = zo.symbol(emit.atom_index).atom(elf_file).?; - const r_type: std.elf.R_X86_64 = if (!emit.pic) - .@"32" - else if (reloc.target.is_extern and !reloc.target.force_pcrel_direct) - .GOTPCREL - else - .PC32; - try atom.addReloc(gpa, .{ - .r_offset = end_offset - 4, - .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), - .r_addend = if (emit.pic) reloc.off - 4 else reloc.off, - }, zo); - } else if (emit.bin_file.cast(.macho)) |macho_file| { - const zo = macho_file.getZigObject().?; - const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?; - try atom.addReloc(macho_file, .{ - .tag = .@"extern", - .offset = end_offset - 4, - .target = reloc.target.index, - .addend = reloc.off, - .type = if (reloc.target.is_extern and !reloc.target.force_pcrel_direct) .got_load else .signed, - .meta = .{ - .pcrel = true, - .has_subtractor = false, - .length = 2, - .symbolnum = @intCast(reloc.target.index), - }, - }); - } else if (emit.bin_file.cast(.elf2)) |elf| try elf.addReloc( - @enumFromInt(emit.atom_index), - end_offset - 4, - @enumFromInt(reloc.target.index), - reloc.off, - .{ .x86_64 = .@"32" }, - ) else if (emit.bin_file.cast(.coff)) |coff_file| { - const atom_index = coff_file.getAtomIndexForSymbol( - .{ .sym_index = emit.atom_index, .file = null }, - ).?; - try coff_file.addRelocation(atom_index, .{ - .type = if (reloc.target.is_extern) .got else .direct, - .target = if (reloc.target.is_extern) - coff_file.getGlobalByIndex(reloc.target.index) - else - .{ .sym_index = reloc.target.index, .file = null }, - .offset = end_offset - 4, - .addend = @intCast(reloc.off), - .pcrel = true, - .length = 2, - }); - } else unreachable, - .branch => if (emit.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const atom = zo.symbol(emit.atom_index).atom(elf_file).?; - const r_type: std.elf.R_X86_64 = .PLT32; - try atom.addReloc(gpa, .{ - .r_offset = end_offset - 4, - .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), - .r_addend = reloc.off - 4, - }, zo); - } else if (emit.bin_file.cast(.elf2)) |elf| try elf.addReloc( - @enumFromInt(emit.atom_index), - end_offset - 4, - @enumFromInt(reloc.target.index), - reloc.off - 4, - .{ .x86_64 = .PC32 }, - ) else if (emit.bin_file.cast(.macho)) |macho_file| { - const zo = macho_file.getZigObject().?; - const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?; - try atom.addReloc(macho_file, .{ - .tag = .@"extern", - .offset = end_offset - 4, - .target = reloc.target.index, - .addend = reloc.off, - .type = .branch, - .meta = .{ - .pcrel = true, - .has_subtractor = false, - .length = 2, - .symbolnum = @intCast(reloc.target.index), - }, - }); - } else if (emit.bin_file.cast(.coff)) |coff_file| { - const atom_index = coff_file.getAtomIndexForSymbol( - .{ .sym_index = emit.atom_index, .file = null }, - ).?; - try coff_file.addRelocation(atom_index, .{ - .type = if (reloc.target.is_extern) .import else .got, - .target = if (reloc.target.is_extern) - coff_file.getGlobalByIndex(reloc.target.index) - else - .{ .sym_index = reloc.target.index, .file = null }, - .offset = end_offset - 4, - .addend = @intCast(reloc.off), - .pcrel = true, - .length = 2, - }); - } else return emit.fail("TODO implement {s} reloc for {s}", .{ - @tagName(reloc.target.type), @tagName(emit.bin_file.tag), - }), - .tls => if (emit.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const atom = zo.symbol(emit.atom_index).atom(elf_file).?; - const r_type: std.elf.R_X86_64 = if (emit.pic) .TLSLD else unreachable; - try atom.addReloc(gpa, .{ - .r_offset = end_offset - 4, - .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), - .r_addend = reloc.off - 4, - }, zo); - } else return emit.fail("TODO implement {s} reloc for {s}", .{ - @tagName(reloc.target.type), @tagName(emit.bin_file.tag), - }), - .tlv => if (emit.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const atom = zo.symbol(emit.atom_index).atom(elf_file).?; - const r_type: std.elf.R_X86_64 = if (emit.pic) .DTPOFF32 else .TPOFF32; - try atom.addReloc(gpa, .{ - .r_offset = end_offset - 4, - .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), - .r_addend = reloc.off, - }, zo); - } else if (emit.bin_file.cast(.elf2)) |elf| try elf.addReloc( - @enumFromInt(emit.atom_index), - end_offset - 4, - @enumFromInt(reloc.target.index), - reloc.off, - .{ .x86_64 = .TPOFF32 }, - ) else if (emit.bin_file.cast(.macho)) |macho_file| { - const zo = macho_file.getZigObject().?; - const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?; - try atom.addReloc(macho_file, .{ - .tag = .@"extern", - .offset = end_offset - 4, - .target = reloc.target.index, - .addend = reloc.off, - .type = .tlv, - .meta = .{ - .pcrel = true, - .has_subtractor = false, - .length = 2, - .symbolnum = @intCast(reloc.target.index), - }, - }); - } else return emit.fail("TODO implement {s} reloc for {s}", .{ - @tagName(reloc.target.type), @tagName(emit.bin_file.tag), - }), - }; -} - -fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error { - return switch (emit.lower.fail(format, args)) { - error.LowerFail => error.EmitFail, - else => |e| e, - }; -} - -const Reloc = struct { - /// Offset of the instruction. - inst_offset: u32, - /// Length of the instruction. - inst_length: u4, - /// Offset of the relocation within the instruction. - source_offset: u4, - /// Length of the relocation. - source_length: u4, - /// Target of the relocation. - target: Mir.Inst.Index, - /// Offset from the target. - target_offset: i32, -}; - -const TableReloc = struct { - /// Offset of the relocation. - source_offset: u32, - /// Offset from the start of the table. - target_offset: i32, -}; - -const Loc = struct { - line: u32, - column: u32, - is_stmt: bool, -}; - -fn dbgAdvancePCAndLine(emit: *Emit, loc: Loc) Error!void { - const delta_line = @as(i33, loc.line) - @as(i33, emit.prev_di_loc.line); - const delta_pc: usize = emit.w.end - emit.prev_di_pc; - log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line }); - switch (emit.debug_output) { - .dwarf => |dwarf| { - if (loc.is_stmt != emit.prev_di_loc.is_stmt) try dwarf.negateStmt(); - if (loc.column != emit.prev_di_loc.column) try dwarf.setColumn(loc.column); - try dwarf.advancePCAndLine(delta_line, delta_pc); - emit.prev_di_loc = loc; - emit.prev_di_pc = emit.w.end; - }, - .none => {}, - } -} - -const assert = std.debug.assert; -const bits = @import("bits.zig"); -const codegen = @import("../../codegen.zig"); -const Emit = @This(); -const encoder = @import("encoder.zig"); -const Instruction = encoder.Instruction; -const InternPool = @import("../../InternPool.zig"); -const link = @import("../../link.zig"); -const log = std.log.scoped(.emit); -const Lower = @import("Lower.zig"); -const Mir = @import("Mir.zig"); -const std = @import("std"); -const Type = @import("../../Type.zig"); -const Zcu = @import("../../Zcu.zig"); diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig deleted file mode 100644 index 3b41e0b07afdc16b5136aeb0ee4d89ab784a2572..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/Encoding.zig +++ /dev/null @@ -1,1072 +0,0 @@ -const Encoding = @This(); - -const std = @import("std"); -const assert = std.debug.assert; -const math = std.math; - -const bits = @import("bits.zig"); -const encoder = @import("encoder.zig"); -const Instruction = encoder.Instruction; -const Operand = Instruction.Operand; -const Prefix = Instruction.Prefix; -const Register = bits.Register; -const Rex = encoder.Rex; -const LegacyPrefixes = encoder.LegacyPrefixes; - -mnemonic: Mnemonic, -data: Data, - -const Data = struct { - op_en: OpEn, - ops: [4]Op, - opc_len: u3, - opc: [7]u8, - modrm_ext: u3, - mode: Mode, - feature: Feature, -}; - -pub fn findByMnemonic( - prefix: Instruction.Prefix, - mnemonic: Mnemonic, - ops: []const Instruction.Operand, - target: *const std.Target, -) !?Encoding { - var input_ops: [4]Op = @splat(.none); - for (input_ops[0..ops.len], ops) |*input_op, op| input_op.* = Op.fromOperand(op, target); - - const rex_required = for (ops) |op| switch (op) { - .reg => |r| switch (r) { - .spl, .bpl, .sil, .dil => break true, - else => {}, - }, - else => {}, - } else false; - const rex_invalid = for (ops) |op| switch (op) { - .reg => |r| switch (r) { - .ah, .bh, .ch, .dh => break true, - else => {}, - }, - else => {}, - } else false; - const rex_extended = for (ops) |op| { - if (op.baseExtEnc() != 0b00 or op.indexExtEnc() != 0b00) break true; - } else false; - - if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode; - - var shortest_enc: ?Encoding = null; - var shortest_len: ?usize = null; - next: for (mnemonic_to_encodings_map[@intFromEnum(mnemonic)]) |data| { - if (!switch (data.feature) { - .none => true, - .@"32bit" => switch (target.cpu.arch) { - else => unreachable, - .x86 => true, - .x86_64 => false, - }, - .@"64bit" => switch (target.cpu.arch) { - else => unreachable, - .x86 => false, - .x86_64 => true, - }, - inline .@"invpcid 32bit", .@"rdpid 32bit" => |tag| switch (target.cpu.arch) { - else => unreachable, - .x86 => target.cpu.has( - .x86, - @field(std.Target.x86.Feature, @tagName(tag)[0 .. @tagName(tag).len - " 32bit".len]), - ), - .x86_64 => false, - }, - inline .@"invpcid 64bit", .@"rdpid 64bit", .@"prefetchi 64bit" => |tag| switch (target.cpu.arch) { - else => unreachable, - .x86 => false, - .x86_64 => target.cpu.has( - .x86, - @field(std.Target.x86.Feature, @tagName(tag)[0 .. @tagName(tag).len - " 64bit".len]), - ), - }, - .prefetch => target.cpu.hasAny(.x86, &.{ .sse, .prfchw, .prefetchi, .prefetchwt1 }), - inline else => |tag| has_features: { - comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' '); - comptime var features: []const std.Target.x86.Feature = &.{}; - inline while (comptime feature_it.next()) |feature| features = features ++ .{@field(std.Target.x86.Feature, feature)}; - break :has_features target.cpu.hasAll(.x86, features); - }, - }) continue; - - switch (data.mode) { - .none, .short => if (rex_required) continue, - .rex, .rex_short => if (!rex_required) continue, - else => {}, - } - for (input_ops, data.ops) |input_op, data_op| if (!input_op.isSubset(data_op)) continue :next; - - const enc: Encoding = .{ .mnemonic = mnemonic, .data = data }; - if (shortest_enc) |previous_shortest_enc| { - const len = estimateInstructionLength(prefix, enc, ops); - const previous_shortest_len = shortest_len orelse - estimateInstructionLength(prefix, previous_shortest_enc, ops); - if (len < previous_shortest_len) { - shortest_enc = enc; - shortest_len = len; - } else shortest_len = previous_shortest_len; - } else shortest_enc = enc; - } - return shortest_enc; -} - -/// Returns first matching encoding by opcode. -pub fn findByOpcode(opc: []const u8, prefixes: struct { - legacy: LegacyPrefixes, - rex: Rex, -}, modrm_ext: ?u3) ?Encoding { - for (mnemonic_to_encodings_map, 0..) |encs, mnemonic_int| for (encs) |data| { - const enc = Encoding{ .mnemonic = @as(Mnemonic, @enumFromInt(mnemonic_int)), .data = data }; - if (modrm_ext) |ext| if (ext != data.modrm_ext) continue; - if (!std.mem.eql(u8, opc, enc.opcode())) continue; - if (prefixes.rex.w) { - if (!data.mode.isLong()) continue; - } else if (prefixes.rex.present and !prefixes.rex.isSet()) { - if (!data.mode.isRex()) continue; - } else if (prefixes.legacy.prefix_66) { - if (!data.mode.isShort()) continue; - } else { - if (data.mode.isShort()) continue; - } - return enc; - }; - return null; -} - -pub fn opcode(encoding: *const Encoding) []const u8 { - return encoding.data.opc[0..encoding.data.opc_len]; -} - -pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 { - const prefix = encoding.data.opc[0]; - return switch (prefix) { - 0x66, 0xf2, 0xf3 => prefix, - else => null, - }; -} - -pub fn modRmExt(encoding: Encoding) u3 { - return switch (encoding.data.op_en) { - .ia, .m, .mi, .m1, .mc, .vm, .vmi => encoding.data.modrm_ext, - else => unreachable, - }; -} - -pub fn format(encoding: Encoding, writer: *std.Io.Writer) std.Io.Writer.Error!void { - var opc = encoding.opcode(); - if (encoding.data.mode.isVex()) { - try writer.writeAll("VEX."); - - try writer.writeAll(switch (encoding.data.mode) { - .vex_128_w0, .vex_128_w1, .vex_128_wig => "128", - .vex_256_w0, .vex_256_w1, .vex_256_wig => "256", - .vex_lig_w0, .vex_lig_w1, .vex_lig_wig => "LIG", - .vex_lz_w0, .vex_lz_w1, .vex_lz_wig => "LZ", - else => unreachable, - }); - - switch (opc[0]) { - else => {}, - 0x66, 0xf3, 0xf2 => { - try writer.print(".{X:0>2}", .{opc[0]}); - opc = opc[1..]; - }, - } - - try writer.print(".{X}", .{opc[0 .. opc.len - 1]}); - opc = opc[opc.len - 1 ..]; - - try writer.writeAll(".W"); - try writer.writeAll(switch (encoding.data.mode) { - .vex_128_w0, .vex_256_w0, .vex_lig_w0, .vex_lz_w0 => "0", - .vex_128_w1, .vex_256_w1, .vex_lig_w1, .vex_lz_w1 => "1", - .vex_128_wig, .vex_256_wig, .vex_lig_wig, .vex_lz_wig => "IG", - else => unreachable, - }); - - try writer.writeByte(' '); - } else if (encoding.data.mode.isLong()) try writer.writeAll("REX.W + "); - for (opc) |byte| try writer.print("{x:0>2} ", .{byte}); - - switch (encoding.data.op_en) { - .z, .fd, .td, .i, .zi, .ii, .d => {}, - .o, .zo, .oz, .oi => { - const op = switch (encoding.data.op_en) { - .o, .oz, .oi => encoding.data.ops[0], - .zo => encoding.data.ops[1], - else => unreachable, - }; - const tag = switch (op) { - .r8 => "rb", - .r16 => "rw", - .r32 => "rd", - .r64 => "rd", - else => unreachable, - }; - try writer.print("+{s} ", .{tag}); - }, - .ia, .m, .mi, .m1, .mc, .vm, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}), - .mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr, .rmv => try writer.writeAll("/r "), - } - - switch (encoding.data.op_en) { - .i, .d, .zi, .ii, .ia, .oi, .mi, .rmi, .mri, .vmi, .rvmi => for (0..2) |i| { - const op = switch (i) { - 0 => switch (encoding.data.op_en) { - .i, .ii, .ia, .d => encoding.data.ops[0], - .zi, .oi, .mi => encoding.data.ops[1], - .rmi, .mri, .vmi => encoding.data.ops[2], - .rvmi => encoding.data.ops[3], - else => unreachable, - }, - 1 => switch (encoding.data.op_en) { - .ii => encoding.data.ops[1], - else => break, - }, - else => unreachable, - }; - const tag = switch (op) { - .imm8, .imm8s => "ib", - .imm16, .imm16s => "iw", - .imm32, .imm32s => "id", - .imm64 => "io", - .rel8 => "cb", - .rel16 => "cw", - .rel32 => "cd", - else => unreachable, - }; - try writer.print("{s} ", .{tag}); - }, - .rvmr => try writer.writeAll("/is4 "), - .z, .fd, .td, .o, .zo, .oz, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .vm, .rvm, .mvr, .rmv => {}, - } - - try writer.print("{s} ", .{@tagName(encoding.mnemonic)}); - - for (encoding.data.ops) |op| switch (op) { - .none => break, - else => try writer.print("{s} ", .{@tagName(op)}), - }; - - const op_en = switch (encoding.data.op_en) { - .zi => .i, - else => |op_en| op_en, - }; - try writer.print("{s}", .{@tagName(op_en)}); -} - -pub const Mnemonic = enum { - // Directives - @".cfi_def_cfa", - @".cfi_def_cfa_register", - @".cfi_def_cfa_offset", - @".cfi_adjust_cfa_offset", - @".cfi_offset", - @".cfi_val_offset", - @".cfi_rel_offset", - @".cfi_register", - @".cfi_restore", - @".cfi_undefined", - @".cfi_same_value", - @".cfi_remember_state", - @".cfi_restore_state", - @".cfi_escape", - // zig fmt: off - // General-purpose - aaa, aad, aam, aas, adc, add, @"and", arpl, - bound, bsf, bsr, bswap, bt, btc, btr, bts, - call, cbw, cdq, cdqe, - clac, clc, cld, cldemote, clflush, clflushopt, cli, clts, clui, clrssbsy, clwb, cmc, - cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna, - cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno, - cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz, - cmp, cmps, cmpsb, cmpsd, cmpsq, cmpsw, cmpxchg, cmpxchg8b, cmpxchg16b, - cpuid, cqo, cwd, cwde, - daa, das, dec, div, - endbr32, endbr64, enqcmd, enqcmds, enter, - hlt, hreset, - idiv, imul, in, inc, incsspd, incsspq, ins, insb, insd, insw, - int, int1, int3, into, invd, invlpg, invpcid, iret, iretd, iretq, iretw, - ja, jae, jb, jbe, jc, jcxz, je, jecxz, jg, jge, jl, jle, jmp, jna, jnae, jnb, jnbe, - jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, jrcxz, js, jz, - lahf, lar, lea, leave, lfence, lgdt, lidt, lldt, lmsw, loop, loope, loopne, - lods, lodsb, lodsd, lodsq, lodsw, - lsl, ltr, - mfence, mov, movbe, - movs, movsb, movsd, movsq, movsw, - movsx, movsxd, movzx, mul, - neg, nop, not, - @"or", out, outs, outsb, outsd, outsw, - pause, pop, popf, popfd, popfq, push, pushfq, - rcl, rcr, - rdfsbase, rdgsbase, rdmsr, rdpid, rdpkru, rdpmc, rdrand, rdseed, rdsspd, rdsspq, rdtsc, rdtscp, - ret, rol, ror, rsm, - sahf, sal, sar, sbb, - scas, scasb, scasd, scasq, scasw, - senduipi, serialize, - shl, shld, shr, shrd, - stac, stc, std, sti, str, stui, - sub, swapgs, syscall, sysenter, sysexit, sysret, - seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae, - setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns, - setnz, seto, setp, setpe, setpo, sets, setz, - sfence, sidt, sldt, smsw, - stos, stosb, stosd, stosq, stosw, - @"test", testui, tpause, - ud0, ud1, ud2, uiret, umonitor, umwait, - verr, verw, wrfsbase, wrgsbase, wrmsr, wrpkru, wrssd, wrssq, wrussd, wrussq, - xadd, xchg, xgetbv, xlat, xlatb, xor, - // X87 - f2xm1, fabs, fadd, faddp, fbld, fbstp, fchs, fclex, - fcmovb, fcmovbe, fcmove, fcmovnb, fcmovnbe, fcmovne, fcmovnu, fcmovu, - fcom, fcomi, fcomip, fcomp, fcompp, fcos, - fdecstp, fdiv, fdivp, fdivr, fdivrp, ffree, - fiadd, ficom, ficomp, fidiv, fidivr, fild, fimul, fincstp, finit, - fist, fistp, fisub, fisubr, - fld, fld1, fldcw, fldenv, fldl2e, fldl2t, fldlg2, fldln2, fldpi, fldz, - fmul, fmulp, - fnclex, fninit, fnop, fnsave, fnstcw, fnstenv, fnstsw, - fpatan, fprem, fprem1, fptan, frndint, frstor, - fsave, fscale, fsin, fsincos, fsqrt, - fst, fstcw, fstenv, fstp, fstsw, - fsub, fsubp, fsubr, fsubrp, - ftst, fucom, fucomi, fucomip, fucomp, fucompp, - fwait, fxam, fxch, fxtract, fyl2x, fyl2xp1, wait, - // MMX - emms, movd, movq, - packssdw, packsswb, packuswb, - paddb, paddd, paddsb, paddsw, paddusb, paddusw, paddw, - pand, pandn, por, pxor, - pcmpeqb, pcmpeqd, pcmpeqw, - pcmpgtb, pcmpgtd, pcmpgtw, - pmaddwd, pmulhw, pmullw, - pslld, psllq, psllw, - psrad, psraw, - psrld, psrlq, psrlw, - psubb, psubd, psubsb, psubsw, psubusb, psubusw, psubw, - // SSE - addps, addss, - andnps, andps, - cmpps, cmpss, comiss, - cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si, - divps, divss, - fxrstor, fxrstor64, fxsave, fxsave64, - ldmxcsr, - maxps, maxss, - minps, minss, - movaps, movhlps, movhps, movlhps, movlps, - movmskps, - movss, movups, - mulps, mulss, - orps, - pavgb, pavgw, - pextrw, pinsrw, - pmaxsw, pmaxub, pminsw, pminub, pmovmskb, pmulhuw, - prefetchit0, prefetchit1, prefetchnta, prefetcht0, prefetcht1, prefetcht2, prefetchw, prefetchwt1, - psadbw, pshufw, - shufps, - sqrtps, sqrtss, - stmxcsr, - subps, subss, - ucomiss, unpckhps, unpcklps, - xorps, - // SSE2 - addpd, addsd, - andpd, - andnpd, - cmppd, //cmpsd, - comisd, - cvtdq2pd, cvtdq2ps, cvtpd2dq, cvtpd2pi, cvtpd2ps, cvtpi2pd, - cvtps2dq, cvtps2pd, cvtsd2si, cvtsd2ss, cvtsi2sd, cvtss2sd, - cvttpd2dq, cvttpd2pi, cvttps2dq, cvttsd2si, - divpd, divsd, - gf2p8affineinvqb, gf2p8affineqb, gf2p8mulb, - maxpd, maxsd, - minpd, minsd, - movapd, - movdq2q, movdqa, movdqu, - movhpd, movlpd, - movmskpd, movq2dq, - //movsd, - movupd, - mulpd, mulsd, - orpd, - paddq, pmuludq, - pshufd, pshufhw, pshuflw, - pslldq, psrldq, psubq, - punpckhbw, punpckhdq, punpckhqdq, punpckhwd, - punpcklbw, punpckldq, punpcklqdq, punpcklwd, - shufpd, - sqrtpd, sqrtsd, - subpd, subsd, - ucomisd, unpckhpd, unpcklpd, - xorpd, - // SSE3 - addsubpd, addsubps, - fisttp, - haddpd, haddps, - hsubpd, hsubps, - lddqu, - movddup, movshdup, movsldup, - // SSSE3 - pabsb, pabsd, pabsw, palignr, - phaddw, phaddsw, phaddd, phsubw, phsubsw, phsubd, - pmaddubsw, pmulhrsw, pshufb, - psignb, psignd, psignw, - // SSE4.1 - blendpd, blendps, blendvpd, blendvps, - dppd, dpps, - extractps, - insertps, - packusdw, - pblendvb, pblendw, - pcmpeqq, - pextrb, pextrd, pextrq, - phminposuw, - pinsrb, pinsrd, pinsrq, - pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw, - pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq, - pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq, - pmuldq, pmulld, - ptest, - roundpd, roundps, roundsd, roundss, - // SSE4.2 - crc32, pcmpgtq, - // ABM - lzcnt, popcnt, - // PCLMUL - pclmulqdq, - // AES - aesdec, aesdeclast, aesenc, aesenclast, aesimc, aeskeygenassist, - // SHA - sha1rnds4, sha1nexte, sha1msg1, sha1msg2, sha256msg1, sha256msg2, sha256rnds2, - // AVX - vaddpd, vaddps, vaddsd, vaddss, vaddsubpd, vaddsubps, - vaesdec, vaesdeclast, vaesenc, vaesenclast, vaesimc, vaeskeygenassist, - vandnpd, vandnps, vandpd, vandps, - vblendpd, vblendps, vblendvpd, vblendvps, - vbroadcastf128, vbroadcastsd, vbroadcastss, - vcmppd, vcmpps, vcmpsd, vcmpss, vcomisd, vcomiss, - vcvtdq2pd, vcvtdq2ps, vcvtpd2dq, vcvtpd2ps, - vcvtps2dq, vcvtps2pd, vcvtsd2si, vcvtsd2ss, - vcvtsi2sd, vcvtsi2ss, vcvtss2sd, vcvtss2si, - vcvttpd2dq, vcvttps2dq, vcvttsd2si, vcvttss2si, - vdivpd, vdivps, vdivsd, vdivss, - vdppd, vdpps, - vextractf128, vextractps, - vgf2p8affineinvqb, vgf2p8affineqb, vgf2p8mulb, - vhaddpd, vhaddps, vhsubpd, vhsubps, - vinsertf128, vinsertps, - vlddqu, vldmxcsr, - vmaskmovpd, vmaskmovps, - vmaxpd, vmaxps, vmaxsd, vmaxss, - vminpd, vminps, vminsd, vminss, - vmovapd, vmovaps, - vmovd, - vmovddup, - vmovdqa, vmovdqu, - vmovhlps, vmovhpd, vmovhps, vmovlhps, vmovlpd, vmovlps, - vmovmskpd, vmovmskps, - vmovq, - vmovsd, - vmovshdup, vmovsldup, - vmovss, - vmovupd, vmovups, - vmulpd, vmulps, vmulsd, vmulss, - vorpd, vorps, - vpabsb, vpabsd, vpabsw, - vpackssdw, vpacksswb, vpackusdw, vpackuswb, - vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw, - vpalignr, vpand, vpandn, vpavgb, vpavgw, - vpblendvb, vpblendw, vpclmulqdq, - vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw, - vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw, - vperm2f128, vpermilpd, vpermilps, - vpextrb, vpextrd, vpextrq, vpextrw, - vphaddw, vphaddsw, vphaddd, vphminposuw, vphsubw, vphsubsw, vphsubd, - vpinsrb, vpinsrd, vpinsrq, vpinsrw, - vpmaddubsw, vpmaddwd, - vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw, - vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw, - vpmovmskb, - vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq, - vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq, - vpmuldq, vpmulhrsw, vpmulhuw, vpmulhw, vpmulld, vpmullw, vpmuludq, - vpor, - vpsadbw, vpshufb, vpshufd, vpshufhw, vpshuflw, - vpsignb, vpsignd, vpsignw, - vpslld, vpslldq, vpsllq, vpsllw, - vpsrad, vpsraq, vpsraw, - vpsrld, vpsrldq, vpsrlq, vpsrlw, - vpsubb, vpsubd, vpsubq, vpsubsb, vpsubsw, vpsubusb, vpsubusw, vpsubw, - vptest, - vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd, - vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd, - vpxor, - vroundpd, vroundps, vroundsd, vroundss, - vshufpd, vshufps, - vsqrtpd, vsqrtps, vsqrtsd, vsqrtss, - vstmxcsr, - vsubpd, vsubps, vsubsd, vsubss, - vtestpd, vtestps, - vucomisd, vucomiss, vunpckhpd, vunpckhps, vunpcklpd, vunpcklps, - vxorpd, vxorps, - // BMI - andn, bextr, blsi, blsmsk, blsr, tzcnt, - // BMI2 - bzhi, mulx, pdep, pext, rorx, sarx, shlx, shrx, - // F16C - vcvtph2ps, vcvtps2ph, - // FMA - vfmadd132pd, vfmadd213pd, vfmadd231pd, - vfmadd132ps, vfmadd213ps, vfmadd231ps, - vfmadd132sd, vfmadd213sd, vfmadd231sd, - vfmadd132ss, vfmadd213ss, vfmadd231ss, - // AVX2 - vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw, - vextracti128, vinserti128, vpblendd, - vperm2i128, vpermd, vpermpd, vpermps, vpermq, - vpmaskmovd, vpmaskmovq, - vpsllvd, vpsllvq, vpsravd, vpsrlvd, vpsrlvq, - // ADX - adcx, adox, - // AESKLE - aesdec128kl, aesdec256kl, aesenc128kl, aesenc256kl, encodekey128, encodekey256, loadiwkey, - // AESKLEWIDE_KL - aesdecwide128kl, aesdecwide256kl, aesencwide128kl, aesencwide256kl, - // zig fmt: on -}; - -pub const OpEn = enum { - // zig fmt: off - z, - o, zo, oz, oi, - i, zi, ii, ia, - d, m, - fd, td, - m1, mc, mi, mr, rm, - rmi, mri, mrc, - rm0, vm, vmi, rvm, rvmr, rvmi, mvr, rmv, - // zig fmt: on -}; - -pub const Op = enum { - // zig fmt: off - none, - unity, - imm8, imm16, imm32, imm64, - imm8s, imm16s, imm32s, - al, ax, eax, rax, - cl, dx, - rip, eip, ip, - r8, r16, r32, r64, - rm8, rm16, rm32, rm64, - r32_m8, r32_m16, r64_m16, - m8, m16, m32, m64, m80, m128, m256, - rel8, rel16, rel32, - m, moffs, mrip8, - sreg, - st0, st, mm, mm_m64, - xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128, - ymm, ymm_m256, - cr, dr, - // zig fmt: on - - pub fn fromOperand(operand: Instruction.Operand, target: *const std.Target) Op { - return switch (operand) { - .none => .none, - - .reg => |reg| switch (reg.class()) { - .general_purpose => switch (reg) { - .al => .al, - .ax => .ax, - .eax => .eax, - .rax => .rax, - .cl => .cl, - .dx => .dx, - else => switch (reg.size().bitSize(target)) { - 8 => .r8, - 16 => .r16, - 32 => .r32, - 64 => .r64, - else => unreachable, - }, - }, - .gphi => .r8, - .segment => .sreg, - .x87 => switch (reg) { - .st0 => .st0, - else => .st, - }, - .mmx => .mm, - .sse => switch (reg) { - .xmm0 => .xmm0, - else => switch (reg.size().bitSize(target)) { - 128 => .xmm, - 256 => .ymm, - else => unreachable, - }, - }, - .ip => switch (reg) { - .rip => .rip, - .eip => .eip, - .ip => .ip, - else => unreachable, - }, - .cr => .cr, - .dr => .dr, - }, - - .mem => |mem| switch (mem) { - .moffs => .moffs, - .sib => switch (mem.bitSize(target)) { - 0 => .m, - 8 => .m8, - 16 => .m16, - 32 => .m32, - 64 => .m64, - 80 => .m80, - 128 => .m128, - 256 => .m256, - else => unreachable, - }, - .rip => switch (mem.bitSize(target)) { - 0, 8 => .mrip8, - 16 => .m16, - 32 => .m32, - 64 => .m64, - 80 => .m80, - 128 => .m128, - 256 => .m256, - else => unreachable, - }, - }, - - .imm => |imm| switch (imm) { - .signed => |x| if (x == 1) - .unity - else if (math.cast(i8, x)) |_| - .imm8s - else if (math.cast(i16, x)) |_| - .imm16s - else - .imm32s, - .unsigned => |x| if (x == 1) - .unity - else if (math.cast(i8, x)) |_| - .imm8s - else if (math.cast(u8, x)) |_| - .imm8 - else if (math.cast(i16, x)) |_| - .imm16s - else if (math.cast(u16, x)) |_| - .imm16 - else if (math.cast(i32, x)) |_| - .imm32s - else if (math.cast(u32, x)) |_| - .imm32 - else - .imm64, - }, - - .bytes => unreachable, - }; - } - - pub fn toReg(op: Op) Register { - return switch (op) { - else => .none, - .al => .al, - .ax => .ax, - .eax => .eax, - .rax => .rax, - .cl => .cl, - .dx => .dx, - .rip => .rip, - .eip => .eip, - .ip => .ip, - .st0 => .st0, - .xmm0 => .xmm0, - }; - } - - pub fn immBitSize(op: Op) u64 { - return switch (op) { - .none, .m, .moffs, .mrip8, .sreg => unreachable, - .al, .cl, .dx, .rip, .eip, .ip, .r8, .rm8, .r32_m8 => unreachable, - .ax, .r16, .rm16 => unreachable, - .eax, .r32, .rm32, .r32_m16 => unreachable, - .rax, .r64, .rm64, .r64_m16 => unreachable, - .st0, .st, .mm, .mm_m64 => unreachable, - .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable, - .ymm, .ymm_m256 => unreachable, - .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable, - .cr, .dr => unreachable, - .unity => 1, - .imm8, .imm8s, .rel8 => 8, - .imm16, .imm16s, .rel16 => 16, - .imm32, .imm32s, .rel32 => 32, - .imm64 => 64, - }; - } - - pub fn regBitSize(op: Op) u64 { - return switch (op) { - .none, .m, .moffs, .mrip8, .sreg => unreachable, - .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable, - .rel8, .rel16, .rel32 => unreachable, - .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable, - .al, .cl, .r8, .rm8 => 8, - .ax, .dx, .ip, .r16, .rm16 => 16, - .eax, .eip, .r32, .rm32, .r32_m8, .r32_m16 => 32, - .rax, .rip, .r64, .rm64, .r64_m16, .mm, .mm_m64, .cr, .dr => 64, - .st0, .st => 80, - .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => 128, - .ymm, .ymm_m256 => 256, - }; - } - - pub fn memBitSize(op: Op) u64 { - return switch (op) { - .none, .m, .moffs, .sreg => unreachable, - .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable, - .rel8, .rel16, .rel32 => unreachable, - .al, .cl, .r8, .ax, .dx, .ip, .r16, .eax, .eip, .r32, .rax, .rip, .r64 => unreachable, - .st0, .st, .mm, .xmm0, .xmm, .ymm => unreachable, - .cr, .dr => unreachable, - .mrip8, .m8, .rm8, .r32_m8, .xmm_m8 => 8, - .m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16, - .m32, .rm32, .xmm_m32 => 32, - .m64, .rm64, .mm_m64, .xmm_m64 => 64, - .m80 => 80, - .m128, .xmm_m128 => 128, - .m256, .ymm_m256 => 256, - }; - } - - pub fn isSigned(op: Op) bool { - return switch (op) { - .unity, .imm8, .imm16, .imm32, .imm64 => false, - .imm8s, .imm16s, .imm32s => true, - .rel8, .rel16, .rel32 => true, - else => unreachable, - }; - } - - pub fn isUnsigned(op: Op) bool { - return !op.isSigned(); - } - - pub fn isRegister(op: Op) bool { - // zig fmt: off - return switch (op) { - .al, .ax, .eax, .rax, - .cl, .dx, - .ip, .eip, .rip, - .r8, .r16, .r32, .r64, - .rm8, .rm16, .rm32, .rm64, - .r32_m8, .r32_m16, .r64_m16, - .st0, .st, .mm, .mm_m64, - .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128, - .ymm, .ymm_m256, - .cr, .dr, - => true, - else => false, - }; - // zig fmt: on - } - - pub fn isImmediate(op: Op) bool { - // zig fmt: off - return switch (op) { - .imm8, .imm16, .imm32, .imm64, - .imm8s, .imm16s, .imm32s, - .rel8, .rel16, .rel32, - .unity, - => true, - else => false, - }; - // zig fmt: on - } - - pub fn isMemory(op: Op) bool { - // zig fmt: off - return switch (op) { - .rm8, .rm16, .rm32, .rm64, - .r32_m8, .r32_m16, .r64_m16, - .m8, .m16, .m32, .m64, .m80, .m128, .m256, - .m, .moffs, .mrip8, - .mm_m64, - .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128, - .ymm_m256, - => true, - else => false, - }; - // zig fmt: on - } - - pub fn isSegmentRegister(op: Op) bool { - return switch (op) { - .moffs, .sreg => true, - else => false, - }; - } - - pub fn class(op: Op) bits.Register.Class { - return switch (op) { - else => unreachable, - .al, .ax, .eax, .rax, .cl, .dx => .general_purpose, - .r8, .r16, .r32, .r64 => .general_purpose, - .rm8, .rm16, .rm32, .rm64 => .general_purpose, - .r32_m8, .r32_m16, .r64_m16 => .general_purpose, - .sreg => .segment, - .st0, .st => .x87, - .mm, .mm_m64 => .mmx, - .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => .sse, - .ymm, .ymm_m256 => .sse, - .rip, .eip, .ip => .ip, - .cr => .cr, - .dr => .dr, - }; - } - - /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding. - pub fn isSubset(op: Op, target: Op) bool { - switch (op) { - .none, .m, .moffs, .sreg => return op == target, - else => { - if (op.isRegister() and target.isRegister()) { - return switch (target.toReg()) { - .none => op.class() == target.class() and op.regBitSize() == target.regBitSize(), - else => op == target, - }; - } - if (op.isMemory() and target.isMemory()) { - switch (target) { - .m => return true, - .moffs, .mrip8 => return op == target, - else => return op.memBitSize() == target.memBitSize(), - } - } - if (op.isImmediate() and target.isImmediate()) { - switch (target) { - .imm64 => if (op.immBitSize() <= 64) return true, - .imm32s, .rel32 => if (op.immBitSize() < 32 or (op.immBitSize() == 32 and op.isSigned())) - return true, - .imm32 => if (op.immBitSize() <= 32) return true, - .imm16s, .rel16 => if (op.immBitSize() < 16 or (op.immBitSize() == 16 and op.isSigned())) - return true, - .imm16 => if (op.immBitSize() <= 16) return true, - .imm8s, .rel8 => if (op.immBitSize() < 8 or (op.immBitSize() == 8 and op.isSigned())) - return true, - .imm8 => if (op.immBitSize() <= 8) return true, - else => {}, - } - return op == target; - } - return false; - }, - } - } -}; - -pub const Mode = enum { - // zig fmt: off - none, - short, long, - rex, rex_short, - wait, - vex_128_w0, vex_128_w1, vex_128_wig, - vex_256_w0, vex_256_w1, vex_256_wig, - vex_lig_w0, vex_lig_w1, vex_lig_wig, - vex_lz_w0, vex_lz_w1, vex_lz_wig, - // zig fmt: on - - pub fn isShort(mode: Mode) bool { - return switch (mode) { - .short, .rex_short => true, - else => false, - }; - } - - pub fn isLong(mode: Mode) bool { - return switch (mode) { - .long, - .vex_128_w1, - .vex_256_w1, - .vex_lig_w1, - .vex_lz_w1, - => true, - else => false, - }; - } - - pub fn isRex(mode: Mode) bool { - return switch (mode) { - else => false, - .rex, .rex_short => true, - }; - } - - pub fn isVex(mode: Mode) bool { - return switch (mode) { - // zig fmt: off - else => false, - .vex_128_w0, .vex_128_w1, .vex_128_wig, - .vex_256_w0, .vex_256_w1, .vex_256_wig, - .vex_lig_w0, .vex_lig_w1, .vex_lig_wig, - .vex_lz_w0, .vex_lz_w1, .vex_lz_wig, - => true, - // zig fmt: on - }; - } - - pub fn isVecLong(mode: Mode) bool { - return switch (mode) { - // zig fmt: off - else => unreachable, - .vex_128_w0, .vex_128_w1, .vex_128_wig, - .vex_lig_w0, .vex_lig_w1, .vex_lig_wig, - .vex_lz_w0, .vex_lz_w1, .vex_lz_wig, - => false, - .vex_256_w0, .vex_256_w1, .vex_256_wig, - => true, - // zig fmt: on - }; - } -}; - -pub const Feature = enum { - none, - @"32bit", - @"64bit", - adx, - aes, - @"aes avx", - avx, - avx2, - bmi, - bmi2, - cldemote, - clflushopt, - clwb, - cmov, - @"cmov x87", - crc32, - enqcmd, - f16c, - fma, - fsgsbase, - fxsr, - gfni, - @"gfni avx", - hreset, - @"invpcid 32bit", - @"invpcid 64bit", - kl, - lzcnt, - mmx, - movbe, - pclmul, - @"pclmul avx", - pku, - popcnt, - prefetch, - @"prefetchi 64bit", - prefetchwt1, - prfchw, - rdrnd, - rdseed, - @"rdpid 32bit", - @"rdpid 64bit", - sahf, - serialize, - shstk, - smap, - sse, - sse2, - sse3, - @"sse3 x87", - sse4_1, - sse4_2, - ssse3, - sha, - uintr, - vaes, - vpclmulqdq, - waitpkg, - widekl, - x87, -}; - -fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Operand) usize { - var inst: Instruction = .{ - .prefix = prefix, - .encoding = encoding, - .ops = @splat(.none), - }; - @memcpy(inst.ops[0..ops.len], ops); - - // By using a buffer with maximum length of encoded instruction, we can use - // the `end` field of the Writer for the count. - var buf: [16]u8 = undefined; - var trash: std.Io.Writer.Discarding = .init(&buf); - inst.encode(&trash.writer, .{ - .allow_frame_locs = true, - .allow_symbols = true, - }) catch { - // Since the function signature for encode() does not mention under what - // conditions it can fail, I have changed `unreachable` to `@panic` here. - // This is a TODO item since it indicates this function - // (`estimateInstructionLength`) has the wrong function signature. - @panic("unexpected failure to encode"); - }; - return trash.writer.end; -} - -const mnemonic_to_encodings_map = init: { - @setEvalBranchQuota(5_900); - const ModrmExt = u3; - const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, ModrmExt, Mode, Feature }; - const encodings: []const Entry = @import("encodings.zon"); - - const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len; - var mnemonic_map: [mnemonic_count][]Data = @splat(&.{}); - for (encodings) |entry| mnemonic_map[@intFromEnum(entry[0])].len += 1; - var data_storage: [encodings.len]Data = undefined; - var storage_index: usize = 0; - for (&mnemonic_map) |*value| { - value.ptr = data_storage[storage_index..].ptr; - storage_index += value.len; - } - var mnemonic_index: [mnemonic_count]usize = @splat(0); - const ops_len = @typeInfo(@FieldType(Data, "ops")).array.len; - const opc_len = @typeInfo(@FieldType(Data, "opc")).array.len; - for (encodings) |entry| { - const index = &mnemonic_index[@intFromEnum(entry[0])]; - mnemonic_map[@intFromEnum(entry[0])][index.*] = .{ - .op_en = entry[1], - .ops = (entry[2] ++ .{.none} ** (ops_len - entry[2].len)).*, - .opc_len = entry[3].len, - .opc = (entry[3] ++ .{undefined} ** (opc_len - entry[3].len)).*, - .modrm_ext = entry[4], - .mode = entry[5], - .feature = entry[6], - }; - index.* += 1; - } - const final_storage = data_storage; - var final_map: [mnemonic_count][]const Data = @splat(&.{}); - storage_index = 0; - for (&final_map, mnemonic_map) |*final_value, value| { - final_value.* = final_storage[storage_index..][0..value.len]; - storage_index += value.len; - } - break :init final_map; -}; diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig deleted file mode 100644 index ca1fb1e428b672731bc57477bfe553e0b8ea2450..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/Lower.zig +++ /dev/null @@ -1,689 +0,0 @@ -//! This file contains the functionality for lowering x86_64 MIR to Instructions - -target: *const std.Target, -allocator: std.mem.Allocator, -mir: Mir, -cc: std.builtin.CallingConvention, -err_msg: ?*Zcu.ErrorMsg = null, -src_loc: Zcu.LazySrcLoc, -result_insts_len: ResultInstIndex = undefined, -result_insts: [max_result_insts]Instruction = undefined, -result_relocs_len: ResultRelocIndex = undefined, -result_relocs: [max_result_relocs]Reloc = undefined, - -const max_result_insts = @max( - 1, // non-pseudo instructions - 2, // cmovcc: cmovcc \ cmovcc - 3, // setcc: setcc \ setcc \ logicop - 2, // jcc: jcc \ jcc - pseudo_probe_align_insts, - pseudo_probe_adjust_unrolled_max_insts, - pseudo_probe_adjust_setup_insts, - pseudo_probe_adjust_loop_insts, - abi.zigcc.callee_preserved_regs.len * 2, // push_regs/pop_regs - abi.Win64.callee_preserved_regs.len * 2, // push_regs/pop_regs - abi.SysV.callee_preserved_regs.len * 2, // push_regs/pop_regs -); -const max_result_relocs = @max( - 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea - 2, // jcc: jcc \ jcc - 2, // test \ jcc \ probe \ sub \ jmp - 1, // probe \ sub \ jcc -); - -const ResultInstIndex = std.math.IntFittingRange(0, max_result_insts); -const ResultRelocIndex = std.math.IntFittingRange(0, max_result_relocs); -pub const InstOpIndex = std.math.IntFittingRange( - 0, - @typeInfo(@FieldType(Instruction, "ops")).array.len, -); - -pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp -pub const pseudo_probe_adjust_unrolled_max_insts = - pseudo_probe_adjust_setup_insts + pseudo_probe_adjust_loop_insts; -pub const pseudo_probe_adjust_setup_insts = 2; // mov \ sub -pub const pseudo_probe_adjust_loop_insts = 3; // probe \ sub \ jcc - -pub const Error = error{ - OutOfMemory, - LowerFail, - InvalidInstruction, - CannotEncode, - CodegenFail, -} || codegen.GenerateSymbolError; - -pub const Reloc = struct { - lowered_inst_index: ResultInstIndex, - op_index: InstOpIndex, - target: Target, - off: i32, - - const Target = union(enum) { - inst: Mir.Inst.Index, - table, - nav: InternPool.Nav.Index, - uav: InternPool.Key.Ptr.BaseAddr.Uav, - lazy_sym: link.File.LazySymbol, - extern_func: Mir.NullTerminatedString, - }; -}; - -const Options = struct { allow_frame_locs: bool }; - -/// The returned slice is overwritten by the next call to lowerMir. -pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { - insts: []Instruction, - relocs: []const Reloc, -} { - lower.result_insts = undefined; - lower.result_relocs = undefined; - errdefer lower.result_insts = undefined; - errdefer lower.result_relocs = undefined; - lower.result_insts_len = 0; - lower.result_relocs_len = 0; - defer lower.result_insts_len = undefined; - defer lower.result_relocs_len = undefined; - - const inst = lower.mir.instructions.get(index); - switch (inst.tag) { - else => try lower.generic(inst), - .pseudo => switch (inst.ops) { - .pseudo_cmov_z_and_np_rr => { - assert(inst.data.rr.fixes == ._); - try lower.encode(.none, .cmovnz, &.{ - .{ .reg = inst.data.rr.r2 }, - .{ .reg = inst.data.rr.r1 }, - }); - try lower.encode(.none, .cmovnp, &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }); - }, - .pseudo_cmov_nz_or_p_rr => { - assert(inst.data.rr.fixes == ._); - try lower.encode(.none, .cmovnz, &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }); - try lower.encode(.none, .cmovp, &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }); - }, - .pseudo_cmov_nz_or_p_rm => { - assert(inst.data.rx.fixes == ._); - try lower.encode(.none, .cmovnz, &.{ - .{ .reg = inst.data.rx.r1 }, - .{ .mem = lower.mem(1, inst.data.rx.payload) }, - }); - try lower.encode(.none, .cmovp, &.{ - .{ .reg = inst.data.rx.r1 }, - .{ .mem = lower.mem(1, inst.data.rx.payload) }, - }); - }, - .pseudo_set_z_and_np_r => { - assert(inst.data.rr.fixes == ._); - try lower.encode(.none, .setz, &.{ - .{ .reg = inst.data.rr.r1 }, - }); - try lower.encode(.none, .setnp, &.{ - .{ .reg = inst.data.rr.r2 }, - }); - try lower.encode(.none, .@"and", &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }); - }, - .pseudo_set_z_and_np_m => { - assert(inst.data.rx.fixes == ._); - try lower.encode(.none, .setz, &.{ - .{ .mem = lower.mem(0, inst.data.rx.payload) }, - }); - try lower.encode(.none, .setnp, &.{ - .{ .reg = inst.data.rx.r1 }, - }); - try lower.encode(.none, .@"and", &.{ - .{ .mem = lower.mem(0, inst.data.rx.payload) }, - .{ .reg = inst.data.rx.r1 }, - }); - }, - .pseudo_set_nz_or_p_r => { - assert(inst.data.rr.fixes == ._); - try lower.encode(.none, .setnz, &.{ - .{ .reg = inst.data.rr.r1 }, - }); - try lower.encode(.none, .setp, &.{ - .{ .reg = inst.data.rr.r2 }, - }); - try lower.encode(.none, .@"or", &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }); - }, - .pseudo_set_nz_or_p_m => { - assert(inst.data.rx.fixes == ._); - try lower.encode(.none, .setnz, &.{ - .{ .mem = lower.mem(0, inst.data.rx.payload) }, - }); - try lower.encode(.none, .setp, &.{ - .{ .reg = inst.data.rx.r1 }, - }); - try lower.encode(.none, .@"or", &.{ - .{ .mem = lower.mem(0, inst.data.rx.payload) }, - .{ .reg = inst.data.rx.r1 }, - }); - }, - .pseudo_j_z_and_np_inst => { - assert(inst.data.inst.fixes == ._); - try lower.encode(.none, .jnz, &.{ - .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) }, - }); - try lower.encode(.none, .jnp, &.{ - .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, - }); - }, - .pseudo_j_nz_or_p_inst => { - assert(inst.data.inst.fixes == ._); - try lower.encode(.none, .jnz, &.{ - .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, - }); - try lower.encode(.none, .jp, &.{ - .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, - }); - }, - - .pseudo_probe_align_ri_s => { - try lower.encode(.none, .@"test", &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = .s(@bitCast(inst.data.ri.i)) }, - }); - try lower.encode(.none, .jz, &.{ - .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) }, - }); - try lower.encode(.none, .lea, &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .mem = Memory.initSib(.qword, .{ - .base = .{ .reg = inst.data.ri.r1 }, - .disp = -page_size, - }) }, - }); - try lower.encode(.none, .@"test", &.{ - .{ .mem = Memory.initSib(.dword, .{ - .base = .{ .reg = inst.data.ri.r1 }, - }) }, - .{ .reg = inst.data.ri.r1.to32() }, - }); - try lower.encode(.none, .jmp, &.{ - .{ .imm = lower.reloc(0, .{ .inst = index }, 0) }, - }); - assert(lower.result_insts_len == pseudo_probe_align_insts); - }, - .pseudo_probe_adjust_unrolled_ri_s => { - var offset = page_size; - while (offset < @as(i32, @bitCast(inst.data.ri.i))) : (offset += page_size) { - try lower.encode(.none, .@"test", &.{ - .{ .mem = Memory.initSib(.dword, .{ - .base = .{ .reg = inst.data.ri.r1 }, - .disp = -offset, - }) }, - .{ .reg = inst.data.ri.r1.to32() }, - }); - } - try lower.encode(.none, .sub, &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = .s(@bitCast(inst.data.ri.i)) }, - }); - assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts); - }, - .pseudo_probe_adjust_setup_rri_s => { - try lower.encode(.none, .mov, &.{ - .{ .reg = inst.data.rri.r2.to32() }, - .{ .imm = .s(@bitCast(inst.data.rri.i)) }, - }); - try lower.encode(.none, .sub, &.{ - .{ .reg = inst.data.rri.r1 }, - .{ .reg = inst.data.rri.r2 }, - }); - assert(lower.result_insts_len == pseudo_probe_adjust_setup_insts); - }, - .pseudo_probe_adjust_loop_rr => { - try lower.encode(.none, .@"test", &.{ - .{ .mem = Memory.initSib(.dword, .{ - .base = .{ .reg = inst.data.rr.r1 }, - .scale_index = .{ .scale = 1, .index = inst.data.rr.r2 }, - .disp = -page_size, - }) }, - .{ .reg = inst.data.rr.r1.to32() }, - }); - try lower.encode(.none, .sub, &.{ - .{ .reg = inst.data.rr.r2 }, - .{ .imm = .s(page_size) }, - }); - try lower.encode(.none, .jae, &.{ - .{ .imm = lower.reloc(0, .{ .inst = index }, 0) }, - }); - assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts); - }, - .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst), - .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst), - - .pseudo_cfi_def_cfa_ri_s => try lower.encode(.directive, .@".cfi_def_cfa", &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, - }), - .pseudo_cfi_def_cfa_register_r => try lower.encode(.directive, .@".cfi_def_cfa_register", &.{ - .{ .reg = inst.data.r.r1 }, - }), - .pseudo_cfi_def_cfa_offset_i_s => try lower.encode(.directive, .@".cfi_def_cfa_offset", &.{ - .{ .imm = lower.imm(.i_s, inst.data.i.i) }, - }), - .pseudo_cfi_adjust_cfa_offset_i_s => try lower.encode(.directive, .@".cfi_adjust_cfa_offset", &.{ - .{ .imm = lower.imm(.i_s, inst.data.i.i) }, - }), - .pseudo_cfi_offset_ri_s => try lower.encode(.directive, .@".cfi_offset", &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, - }), - .pseudo_cfi_val_offset_ri_s => try lower.encode(.directive, .@".cfi_val_offset", &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, - }), - .pseudo_cfi_rel_offset_ri_s => try lower.encode(.directive, .@".cfi_rel_offset", &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, - }), - .pseudo_cfi_register_rr => try lower.encode(.directive, .@".cfi_register", &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }), - .pseudo_cfi_restore_r => try lower.encode(.directive, .@".cfi_restore", &.{ - .{ .reg = inst.data.r.r1 }, - }), - .pseudo_cfi_undefined_r => try lower.encode(.directive, .@".cfi_undefined", &.{ - .{ .reg = inst.data.r.r1 }, - }), - .pseudo_cfi_same_value_r => try lower.encode(.directive, .@".cfi_same_value", &.{ - .{ .reg = inst.data.r.r1 }, - }), - .pseudo_cfi_remember_state_none => try lower.encode(.directive, .@".cfi_remember_state", &.{}), - .pseudo_cfi_restore_state_none => try lower.encode(.directive, .@".cfi_restore_state", &.{}), - .pseudo_cfi_escape_bytes => try lower.encode(.directive, .@".cfi_escape", &.{ - .{ .bytes = inst.data.bytes.get(lower.mir) }, - }), - - .pseudo_dbg_prologue_end_none, - .pseudo_dbg_line_stmt_line_column, - .pseudo_dbg_line_line_column, - .pseudo_dbg_epilogue_begin_none, - .pseudo_dbg_enter_block_none, - .pseudo_dbg_leave_block_none, - .pseudo_dbg_enter_inline_func, - .pseudo_dbg_leave_inline_func, - .pseudo_dbg_arg_none, - .pseudo_dbg_arg_i_s, - .pseudo_dbg_arg_i_u, - .pseudo_dbg_arg_i_64, - .pseudo_dbg_arg_ro, - .pseudo_dbg_arg_fa, - .pseudo_dbg_arg_m, - .pseudo_dbg_arg_val, - .pseudo_dbg_var_args_none, - .pseudo_dbg_var_none, - .pseudo_dbg_var_i_s, - .pseudo_dbg_var_i_u, - .pseudo_dbg_var_i_64, - .pseudo_dbg_var_ro, - .pseudo_dbg_var_fa, - .pseudo_dbg_var_m, - .pseudo_dbg_var_val, - - .pseudo_dead_none, - => {}, - else => unreachable, - }, - } - - return .{ - .insts = lower.result_insts[0..lower.result_insts_len], - .relocs = lower.result_relocs[0..lower.result_relocs_len], - }; -} - -pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { - @branchHint(.cold); - assert(lower.err_msg == null); - lower.err_msg = try .create(lower.allocator, lower.src_loc, format, args); - return error.LowerFail; -} - -pub fn imm(lower: *const Lower, ops: Mir.Inst.Ops, i: u32) Immediate { - return switch (ops) { - .rri_s, - .ri_s, - .i_s, - .mi_s, - .rmi_s, - .pseudo_dbg_arg_i_s, - .pseudo_dbg_var_i_s, - => .s(@bitCast(i)), - - .ii, - .ir, - .rrri, - .rri_u, - .ri_u, - .i_u, - .mi_u, - .rmi, - .rmi_u, - .mri, - .rrm, - .rrmi, - .pseudo_dbg_arg_i_u, - .pseudo_dbg_var_i_u, - => .u(i), - - .ri_64, - => .u(lower.mir.extraData(Mir.Imm64, i).data.decode()), - - .pseudo_dbg_arg_i_64, - .pseudo_dbg_var_i_64, - => unreachable, - - else => unreachable, - }; -} - -fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory { - var m = lower.mir.resolveMemoryExtra(payload).decode(); - switch (m) { - .sib => |*sib| switch (sib.base) { - .none, .reg, .frame => {}, - .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed, - .rip_inst => |inst_index| sib.disp = lower.reloc(op_index, .{ .inst = inst_index }, sib.disp).signed, - .nav => |nav| sib.disp = lower.reloc(op_index, .{ .nav = nav }, sib.disp).signed, - .uav => |uav| sib.disp = lower.reloc(op_index, .{ .uav = uav }, sib.disp).signed, - .lazy_sym => |lazy_sym| sib.disp = lower.reloc(op_index, .{ .lazy_sym = lazy_sym }, sib.disp).signed, - .extern_func => |extern_func| sib.disp = lower.reloc(op_index, .{ .extern_func = extern_func }, sib.disp).signed, - }, - else => {}, - } - return m; -} - -fn reloc(lower: *Lower, op_index: InstOpIndex, target: Reloc.Target, off: i32) Immediate { - lower.result_relocs[lower.result_relocs_len] = .{ - .lowered_inst_index = lower.result_insts_len, - .op_index = op_index, - .target = target, - .off = off, - }; - lower.result_relocs_len += 1; - return .s(0); -} - -fn encode(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void { - lower.result_insts[lower.result_insts_len] = try .new(prefix, mnemonic, ops, lower.target); - lower.result_insts_len += 1; -} - -const inst_tags_len = @typeInfo(Mir.Inst.Tag).@"enum".fields.len; -const inst_fixes_len = @typeInfo(Mir.Inst.Fixes).@"enum".fields.len; -/// Lookup table, indexed by `@intFromEnum(inst.tag) * inst_fixes_len + @intFromEnum(fixes)`. -/// The value is the resulting `Mnemonic`, or `null` if the combination is not valid. -const mnemonic_table: [inst_tags_len * inst_fixes_len]?Mnemonic = table: { - @setEvalBranchQuota(80_000); - var table: [inst_tags_len * inst_fixes_len]?Mnemonic = undefined; - for (0..inst_fixes_len) |fixes_i| { - const fixes: Mir.Inst.Fixes = @enumFromInt(fixes_i); - const prefix, const suffix = affix: { - const pattern = if (std.mem.indexOfScalar(u8, @tagName(fixes), ' ')) |i| - @tagName(fixes)[i + 1 ..] - else - @tagName(fixes); - const wildcard_idx = std.mem.indexOfScalar(u8, pattern, '_').?; - break :affix .{ pattern[0..wildcard_idx], pattern[wildcard_idx + 1 ..] }; - }; - for (0..inst_tags_len) |inst_tag_i| { - const inst_tag: Mir.Inst.Tag = @enumFromInt(inst_tag_i); - const name = prefix ++ @tagName(inst_tag) ++ suffix; - const idx = inst_tag_i * inst_fixes_len + fixes_i; - table[idx] = if (@hasField(Mnemonic, name)) @field(Mnemonic, name) else null; - } - } - break :table table; -}; - -fn generic(lower: *Lower, inst: Mir.Inst) Error!void { - @setEvalBranchQuota(2_000); - const fixes = switch (inst.ops) { - .none => inst.data.none.fixes, - .inst => inst.data.inst.fixes, - .i_s, .i_u => inst.data.i.fixes, - .ii => inst.data.ii.fixes, - .r => inst.data.r.fixes, - .rr => inst.data.rr.fixes, - .rrr => inst.data.rrr.fixes, - .rrrr => inst.data.rrrr.fixes, - .rrri => inst.data.rrri.fixes, - .rri_s, .rri_u => inst.data.rri.fixes, - .ri_s, .ri_u, .ri_64, .ir => inst.data.ri.fixes, - .rm, .rmi_s, .rmi_u, .mr => inst.data.rx.fixes, - .mrr, .rrm, .rmr => inst.data.rrx.fixes, - .rmi, .mri => inst.data.rix.fixes, - .rrmr => inst.data.rrrx.fixes, - .rrmi => inst.data.rrix.fixes, - .mi_u, .mi_s => inst.data.x.fixes, - .m => inst.data.x.fixes, - .nav, .uav, .lazy_sym, .extern_func => ._, - else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}), - }; - try lower.encode(switch (fixes) { - inline else => |tag| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) |space| - @field(Prefix, @tagName(tag)[0..space]) - else - .none, - }, mnemonic: { - if (mnemonic_table[@intFromEnum(inst.tag) * inst_fixes_len + @intFromEnum(fixes)]) |mnemonic| { - break :mnemonic mnemonic; - } - // This combination is invalid; make the theoretical mnemonic name and emit an error with it. - const fixes_name = @tagName(fixes); - const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) |i| i + " ".len else 0..]; - const wildcard_index = std.mem.indexOfScalar(u8, pattern, '_').?; - return lower.fail("unsupported mnemonic: '{s}{s}{s}'", .{ - pattern[0..wildcard_index], - @tagName(inst.tag), - pattern[wildcard_index + "_".len ..], - }); - }, switch (inst.ops) { - .none => &.{}, - .inst => &.{ - .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, - }, - .i_s, .i_u => &.{ - .{ .imm = lower.imm(inst.ops, inst.data.i.i) }, - }, - .ii => &.{ - .{ .imm = lower.imm(inst.ops, inst.data.ii.i1) }, - .{ .imm = lower.imm(inst.ops, inst.data.ii.i2) }, - }, - .ir => &.{ - .{ .imm = lower.imm(inst.ops, inst.data.ri.i) }, - .{ .reg = inst.data.ri.r1 }, - }, - .r => &.{ - .{ .reg = inst.data.r.r1 }, - }, - .rr => &.{ - .{ .reg = inst.data.rr.r1 }, - .{ .reg = inst.data.rr.r2 }, - }, - .rrr => &.{ - .{ .reg = inst.data.rrr.r1 }, - .{ .reg = inst.data.rrr.r2 }, - .{ .reg = inst.data.rrr.r3 }, - }, - .rrrr => &.{ - .{ .reg = inst.data.rrrr.r1 }, - .{ .reg = inst.data.rrrr.r2 }, - .{ .reg = inst.data.rrrr.r3 }, - .{ .reg = inst.data.rrrr.r4 }, - }, - .rrri => &.{ - .{ .reg = inst.data.rrri.r1 }, - .{ .reg = inst.data.rrri.r2 }, - .{ .reg = inst.data.rrri.r3 }, - .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) }, - }, - .ri_s, .ri_u, .ri_64 => &.{ - .{ .reg = inst.data.ri.r1 }, - .{ .imm = lower.imm(inst.ops, inst.data.ri.i) }, - }, - .rri_s, .rri_u => &.{ - .{ .reg = inst.data.rri.r1 }, - .{ .reg = inst.data.rri.r2 }, - .{ .imm = lower.imm(inst.ops, inst.data.rri.i) }, - }, - .m => &.{ - .{ .mem = lower.mem(0, inst.data.x.payload) }, - }, - .mi_s, .mi_u => &.{ - .{ .mem = lower.mem(0, inst.data.x.payload + 1) }, - .{ .imm = lower.imm( - inst.ops, - lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm, - ) }, - }, - .rm => &.{ - .{ .reg = inst.data.rx.r1 }, - .{ .mem = lower.mem(1, inst.data.rx.payload) }, - }, - .rmr => &.{ - .{ .reg = inst.data.rrx.r1 }, - .{ .mem = lower.mem(1, inst.data.rrx.payload) }, - .{ .reg = inst.data.rrx.r2 }, - }, - .rmi => &.{ - .{ .reg = inst.data.rix.r1 }, - .{ .mem = lower.mem(1, inst.data.rix.payload) }, - .{ .imm = lower.imm(inst.ops, inst.data.rix.i) }, - }, - .rmi_s, .rmi_u => &.{ - .{ .reg = inst.data.rx.r1 }, - .{ .mem = lower.mem(1, inst.data.rx.payload + 1) }, - .{ .imm = lower.imm( - inst.ops, - lower.mir.extraData(Mir.Imm32, inst.data.rx.payload).data.imm, - ) }, - }, - .mr => &.{ - .{ .mem = lower.mem(0, inst.data.rx.payload) }, - .{ .reg = inst.data.rx.r1 }, - }, - .mrr => &.{ - .{ .mem = lower.mem(0, inst.data.rrx.payload) }, - .{ .reg = inst.data.rrx.r1 }, - .{ .reg = inst.data.rrx.r2 }, - }, - .mri => &.{ - .{ .mem = lower.mem(0, inst.data.rix.payload) }, - .{ .reg = inst.data.rix.r1 }, - .{ .imm = lower.imm(inst.ops, inst.data.rix.i) }, - }, - .rrm => &.{ - .{ .reg = inst.data.rrx.r1 }, - .{ .reg = inst.data.rrx.r2 }, - .{ .mem = lower.mem(2, inst.data.rrx.payload) }, - }, - .rrmr => &.{ - .{ .reg = inst.data.rrrx.r1 }, - .{ .reg = inst.data.rrrx.r2 }, - .{ .mem = lower.mem(2, inst.data.rrrx.payload) }, - .{ .reg = inst.data.rrrx.r3 }, - }, - .rrmi => &.{ - .{ .reg = inst.data.rrix.r1 }, - .{ .reg = inst.data.rrix.r2 }, - .{ .mem = lower.mem(2, inst.data.rrix.payload) }, - .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) }, - }, - .nav => &.{ - .{ .imm = lower.reloc(0, .{ .nav = inst.data.nav.index }, inst.data.nav.off) }, - }, - .uav => &.{ - .{ .imm = lower.reloc(0, .{ .uav = inst.data.uav }, 0) }, - }, - .lazy_sym => &.{ - .{ .imm = lower.reloc(0, .{ .lazy_sym = inst.data.lazy_sym }, 0) }, - }, - .extern_func => &.{ - .{ .imm = lower.reloc(0, .{ .extern_func = inst.data.extern_func }, 0) }, - }, - else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }), - }); -} - -fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void { - const callee_preserved_regs = abi.getCalleePreservedRegs(lower.cc); - var off: i32 = switch (mnemonic) { - .push => 0, - .pop => undefined, - else => unreachable, - }; - { - var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) { - .push => .reverse, - .pop => .forward, - else => unreachable, - } }); - while (it.next()) |i| { - try lower.encode(.none, mnemonic, &.{.{ - .reg = callee_preserved_regs[i], - }}); - switch (mnemonic) { - .push => off -= 8, - .pop => {}, - else => unreachable, - } - } - } - switch (mnemonic) { - .push => { - var it = inst.data.reg_list.iterator(.{}); - while (it.next()) |i| { - try lower.encode(.directive, .@".cfi_rel_offset", &.{ - .{ .reg = callee_preserved_regs[i] }, - .{ .imm = .s(off) }, - }); - off += 8; - } - assert(off == 0); - }, - .pop => {}, - else => unreachable, - } -} - -const page_size: i32 = 1 << 12; - -const abi = @import("abi.zig"); -const assert = std.debug.assert; -const bits = @import("bits.zig"); -const codegen = @import("../../codegen.zig"); -const encoder = @import("encoder.zig"); -const link = @import("../../link.zig"); -const std = @import("std"); - -const Immediate = Instruction.Immediate; -const Instruction = encoder.Instruction; -const InternPool = @import("../../InternPool.zig"); -const Lower = @This(); -const Memory = Instruction.Memory; -const Mir = @import("Mir.zig"); -const Mnemonic = Instruction.Mnemonic; -const Zcu = @import("../../Zcu.zig"); -const Operand = Instruction.Operand; -const Prefix = Instruction.Prefix; -const Register = bits.Register; -const Type = @import("../../Type.zig"); diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig deleted file mode 100644 index caf41ffb392d07b8a455f09181c77ffb8a679d7e..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/Mir.zig +++ /dev/null @@ -1,2131 +0,0 @@ -//! Machine Intermediate Representation. -//! This data is produced by x86_64 Codegen and consumed by x86_64 Isel. -//! These instructions have a 1:1 correspondence with machine code instructions -//! for the target. MIR can be lowered to source-annotated textual assembly code -//! instructions, or it can be lowered to machine code. -//! The main purpose of MIR is to postpone the assignment of offsets until Isel, -//! so that, for example, the smaller encodings of jump instructions can be used. - -instructions: std.MultiArrayList(Inst).Slice, -/// The meaning of this data is determined by `Inst.Tag` value. -extra: []const u32, -string_bytes: []const u8, -locals: []const Local, -table: []const Inst.Index, -/// Optional data which, when present, can be used to accelerate encoding speed. -memoized_encodings: []const u0 = &.{}, -frame_locs: std.MultiArrayList(FrameLoc).Slice, - -pub const Inst = struct { - tag: Tag, - ops: Ops, - data: Data, - - pub const Index = u32; - - pub const Fixes = enum(u8) { - /// ___ - @"_", - - /// ___ 0 - _0, - /// ___ 1 - _1, - /// ___ 2 - _2, - /// ___ 3 - _3, - /// ___ 4 - _4, - - /// ___ Demote - _demote, - /// ___ Flush - _flush, - /// ___ Flush Optimized - _flushopt, - /// ___ Instructions With T0 Hint - _it0, - /// ___ Instructions With T0 Hint - _it1, - /// ___ With NTA Hint - _nta, - /// System Call ___ - sys_, - /// ___ With T0 Hint - _t0, - /// ___ With T1 Hint - _t1, - /// ___ With T2 Hint - _t2, - /// ___ Write Back - _wb, - /// ___ With Intent to Write and T1 Hint - _wt1, - - /// ___ crement Shadow Stack Pointer Doubleword - _csspd, - /// ___ crement Shadow Stack Pointer Quadword - _csspq, - /// ___ FS Segment Base - _fsbase, - /// ___ GS - _gs, - /// ___ GS Segment Base - _gsbase, - /// ___ Model Specific Register - _msr, - /// ___ MXCSR - _mxcsr, - /// ___ Processor ID - _pid, - /// ___ Protection Key Rights For User Pages - _pkru, - /// ___ Performance-Monitoring Counters - _pmc, - /// ___ Random Number - _rand, - /// ___ r Busy Flag in a Supervisor Shadow Stack token - _rssbsy, - /// ___ Random Seed - _seed, - /// ___ Shadow Stack Doubleword - _ssd, - /// ___ Shadow Stack Quadword - _ssq, - /// ___ Shadow Stack Pointer Doubleword - _sspd, - /// ___ Shadow Stack Pointer Quadword - _sspq, - /// ___ Time-Stamp Counter - _tsc, - /// ___ Time-Stamp Counter And Processor ID - _tscp, - /// ___ User Shadow Stack Doubleword - _ussd, - /// ___ User Shadow Stack Quadword - _ussq, - /// VEX-Encoded ___ MXCSR - v_mxcsr, - - /// Byte ___ - b_, - /// Interrupt ___ - /// Integer ___ - i_, - /// Interrupt ___ Word - i_w, - /// Interrupt ___ Doubleword - i_d, - /// Interrupt ___ Quadword - i_q, - /// User-Interrupt ___ - ui_, - - /// ___ mp - _mp, - /// ___ if CX register is 0 - _cxz, - /// ___ if ECX register is 0 - _ecxz, - /// ___ if RCX register is 0 - _rcxz, - - /// ___ Addition - _a, - /// ___ Subtraction - _s, - /// ___ Multiply - _m, - /// ___ Division - _d, - - /// ___ Without Affecting Flags - _x, - /// ___ Left - _l, - /// ___ Left Double - _ld, - /// ___ Left Without Affecting Flags - _lx, - /// ___ Mask - _msk, - /// ___ Right - /// ___ For Reading - /// ___ Register - _r, - /// ___ Right Double - _rd, - /// ___ Right Without Affecting Flags - _rx, - - /// ___ Forward - _f, - /// ___ Reverse - //_r, - - /// ___ Above - //_a, - /// ___ Above Or Equal - _ae, - /// ___ Below - _b, - /// ___ Below Or Equal - /// ___ Big Endian - _be, - /// ___ Carry - /// ___ Carry Flag - _c, - /// ___ Equal - _e, - /// ___ Greater - _g, - /// ___ Greater Or Equal - _ge, - /// ___ Less - //_l, - /// ___ Less Or Equal - _le, - /// ___ Not Above - _na, - /// ___ Not Above Or Equal - _nae, - /// ___ Not Below - _nb, - /// ___ Not Below Or Equal - _nbe, - /// ___ Not Carry - _nc, - /// ___ Not Equal - _ne, - /// ___ Not Greater - _ng, - /// ___ Not Greater Or Equal - _nge, - /// ___ Not Less - _nl, - /// ___ Not Less Or Equal - _nle, - /// ___ Not Overflow - _no, - /// ___ Not Parity - _np, - /// ___ Not Sign - _ns, - /// ___ Not Zero - _nz, - /// ___ Overflow - _o, - /// ___ Parity - _p, - /// ___ Parity Even - _pe, - /// ___ Parity Odd - _po, - /// ___ Sign - //_s, - /// ___ Zero - _z, - /// ___ Alignment Check Flag - _ac, - /// ___ Direction Flag - //_d, - /// ___ Interrupt Flag - _i, - /// ___ Task-Switched Flag In CR0 - _ts, - /// ___ User Interrupt Flag - _ui, - - /// ___ Byte - //_b, - /// ___ Word - /// ___ For Writing - /// ___ With Intent to Write - _w, - /// ___ Doubleword - //_d, - /// ___ Double Quadword to Quadword - _dq2q, - /// ___ QuadWord - _q, - /// ___ Quadword to Double Quadword - _q2dq, - - /// ___ String - //_s, - /// ___ String Byte - _sb, - /// ___ String Word - _sw, - /// ___ String Doubleword - _sd, - /// ___ String Quadword - _sq, - - /// Repeat ___ String - @"rep _s", - /// Repeat ___ String Byte - @"rep _sb", - /// Repeat ___ String Word - @"rep _sw", - /// Repeat ___ String Doubleword - @"rep _sd", - /// Repeat ___ String Quadword - @"rep _sq", - - /// Repeat Equal ___ String - @"repe _s", - /// Repeat Equal ___ String Byte - @"repe _sb", - /// Repeat Equal ___ String Word - @"repe _sw", - /// Repeat Equal ___ String Doubleword - @"repe _sd", - /// Repeat Equal ___ String Quadword - @"repe _sq", - - /// Repeat Not Equal ___ String - @"repne _s", - /// Repeat Not Equal ___ String Byte - @"repne _sb", - /// Repeat Not Equal ___ String Word - @"repne _sw", - /// Repeat Not Equal ___ String Doubleword - @"repne _sd", - /// Repeat Not Equal ___ String Quadword - @"repne _sq", - - /// Repeat Not Zero ___ String - @"repnz _s", - /// Repeat Not Zero ___ String Byte - @"repnz _sb", - /// Repeat Not Zero ___ String Word - @"repnz _sw", - /// Repeat Not Zero ___ String Doubleword - @"repnz _sd", - /// Repeat Not Zero ___ String Quadword - @"repnz _sq", - - /// Repeat Zero ___ String - @"repz _s", - /// Repeat Zero ___ String Byte - @"repz _sb", - /// Repeat Zero ___ String Word - @"repz _sw", - /// Repeat Zero ___ String Doubleword - @"repz _sd", - /// Repeat Zero ___ String Quadword - @"repz _sq", - - /// Locked ___ - @"lock _", - /// ___ And Complement - //_c, - /// Locked ___ And Complement - @"lock _c", - /// ___ And Reset - //_r, - /// Locked ___ And Reset - @"lock _r", - /// ___ And Set - //_s, - /// Locked ___ And Set - @"lock _s", - /// ___ 8 Bytes - _8b, - /// Locked ___ 8 Bytes - @"lock _8b", - /// ___ 16 Bytes - _16b, - /// Locked ___ 16 Bytes - @"lock _16b", - - /// Float ___ - f_, - /// Float ___ +1.0 - /// Float ___ 1 - f_1, - /// Float ___ Below - f_b, - /// Float ___ Below Or Equal - f_be, - /// Float ___ Control Word - f_cw, - /// Float ___ Equal - f_e, - /// Float ___ Environment - f_env, - /// Float ___ log_2(e) - f_l2e, - /// Float ___ log_2(10) - f_l2t, - /// Float ___ log_10(2) - f_lg2, - /// Float ___ log_e(2) - f_ln2, - /// Float ___ Not Below - f_nb, - /// Float ___ Not Below Or Equal - f_nbe, - /// Float ___ Not Equal - f_ne, - /// Float ___ Not Unordered - f_nu, - /// Float ___ Pop - f_p, - /// Float ___ +1 - f_p1, - /// Float ___ π - f_pi, - /// Float ___ Pop Pop - f_pp, - /// Float ___ crement Stack-Top Pointer - f_cstp, - /// Float ___ Status Word - f_sw, - /// Float ___ Unordered - f_u, - /// Float ___ +0.0 - f_z, - /// Float BCD ___ - fb_, - /// Float BCD ___ Pop - fb_p, - /// Float And Integer ___ - fi_, - /// Float And Integer ___ Pop - fi_p, - /// Float No Wait ___ - fn_, - /// Float No Wait ___ Control Word - fn_cw, - /// Float No Wait ___ Environment - fn_env, - /// Float No Wait ___ status word - fn_sw, - /// Float Extended ___ - fx_, - /// Float Extended ___ 64 - fx_64, - - /// ___ in 32-bit and Compatibility Mode - _32, - /// ___ in 64-bit Mode - _64, - - /// Packed ___ - p_, - /// Packed ___ Byte - p_b, - /// Packed ___ Word - p_w, - /// Packed ___ Doubleword - p_d, - /// Packed ___ Quadword - p_q, - /// Packed ___ Double Quadword - /// Packed ___ Doubleword to Quadword - p_dq, - /// Packed ___ Unsigned Doubleword to Quadword - p_udq, - /// Packed Carry-Less ___ Quadword to Double Quadword - pcl_qdq, - /// Packed Half ___ Doubleword - ph_d, - /// Packed Half ___ Saturate Word - ph_sw, - /// Packed Half ___ Word - ph_w, - /// ___ Aligned Packed Integer Values - _dqa, - /// ___ Unaligned Packed Integer Values - _dqu, - - /// ___ Scalar Single-Precision Values - _ss, - /// ___ Packed Single-Precision Values - _ps, - /// ___ Scalar Double-Precision Values - //_sd, - /// ___ Packed Double-Precision Values - _pd, - /// Half ___ Packed Single-Precision Values - h_ps, - /// Half ___ Packed Double-Precision Values - h_pd, - - /// ___ Internal Caches - //_d, - /// ___ TLB Entries - _lpg, - /// ___ Process-Context Identifier - _pcid, - - /// Load ___ - l_, - /// Memory ___ - m_, - /// Store ___ - s_, - /// Timed ___ - t_, - /// User Level Monitor ___ - um_, - - /// VEX-Encoded ___ - v_, - /// VEX-Encoded ___ Byte - v_b, - /// VEX-Encoded ___ Word - v_w, - /// VEX-Encoded ___ Doubleword - v_d, - /// VEX-Encoded ___ Quadword - v_q, - /// VEX-Encoded ___ Aligned Packed Integer Values - v_dqa, - /// VEX-Encoded ___ Unaligned Packed Integer Values - v_dqu, - /// VEX-Encoded ___ Integer Data - v_i128, - /// VEX-Encoded Packed ___ - vp_, - /// VEX-Encoded Packed ___ Byte - vp_b, - /// VEX-Encoded Packed ___ Word - vp_w, - /// VEX-Encoded Packed ___ Doubleword - vp_d, - /// VEX-Encoded Packed ___ Quadword - vp_q, - /// VEX-Encoded Packed ___ Double Quadword - /// VEX-Encoded Packed ___ Doubleword to Quadword - vp_dq, - /// VEX-Encoded Packed ___ Unsigned Doubleword to Quadword - vp_udq, - /// VEx-Encoded Packed Carry-Less ___ Quadword to Double Quadword - vpcl_qdq, - /// VEX-Encoded Packed Half ___ Doubleword - vph_d, - /// VEX-Encoded Packed Half ___ Saturate Word - vph_sw, - /// VEX-Encoded Packed Half ___ Word - vph_w, - /// VEX-Encoded ___ Scalar Single-Precision Values - v_ss, - /// VEX-Encoded ___ Packed Single-Precision Values - v_ps, - /// VEX-Encoded ___ Scalar Double-Precision Values - v_sd, - /// VEX-Encoded ___ Packed Double-Precision Values - v_pd, - /// VEX-Encoded ___ 128-Bits Of Floating-Point Data - v_f128, - /// VEX-Encoded Half ___ Packed Single-Precision Values - vh_ps, - /// VEX-Encoded Half ___ Packed Double-Precision Values - vh_pd, - - /// ___ 128-bit key with key locker - _128, - /// ___ 256-bit key with key locker - _256, - /// ___ with key locker using 128-bit key - _128kl, - /// ___ with key locker using 256-bit key - _256kl, - /// ___ with key locker on 8 blocks using 128-bit key - _wide128kl, - /// ___ with key locker on 8 blocks using 256-bit key - _wide256kl, - - /// Mask ___ Byte - k_b, - /// Mask ___ Word - k_w, - /// Mask ___ Doubleword - k_d, - /// Mask ___ Quadword - k_q, - - pub fn fromCond(cc: bits.Condition) Fixes { - return switch (cc) { - inline else => |cc_tag| @field(Fixes, "_" ++ @tagName(cc_tag)), - .z_and_np, .nz_or_p => unreachable, - }; - } - }; - - pub const Tag = enum(u8) { - // General-purpose - /// ASCII adjust al after addition - /// ASCII adjust ax before division - /// ASCII adjust ax after multiply - /// ASCII adjust al after subtraction - aa, - /// Add with carry - /// Unsigned integer addition of two operands with carry flag - adc, - /// Add - /// Add packed integers - /// Add packed single-precision floating-point values - /// Add scalar single-precision floating-point values - /// Add packed double-precision floating-point values - /// Add scalar double-precision floating-point values - /// Packed single-precision floating-point horizontal add - /// Packed double-precision floating-point horizontal add - /// Packed horizontal add - /// Packed horizontal add and saturate - add, - /// Logical and - /// Bitwise logical and of packed single-precision floating-point values - /// Bitwise logical and of packed double-precision floating-point values - @"and", - /// Adjust RPL field of segment selector - arpl, - /// Bit scan forward - /// Bit scan reverse - bs, - /// Byte swap - /// Swap GS base register - swap, - /// Bit test - /// Bit test and complement - /// Bit test and reset - /// Bit test and set - bt, - /// Check array index against bounds - bound, - /// Call - /// Fast system call - call, - /// Convert byte to word - cbw, - /// Convert doubleword to quadword - cdq, - /// Convert doubleword to quadword - cdqe, - /// Clear AC flag in EFLAGS register - /// Clear carry flag - /// Clear direction flag - /// Clear interrupt flag - /// Clear task-switched flag in CR0 - /// Clear user interrupt flag - /// Cache line demote - /// Flush cache line - /// Flush cache line optimized - /// Clear busy flag in a supervisor shadow stack token - /// Cache line write back - cl, - /// Complement carry flag - cmc, - /// Conditional move - cmov, - /// Logical compare - /// Compare string - /// Compare scalar single-precision floating-point values - /// Compare scalar double-precision floating-point values - cmp, - /// Compare and exchange - /// Compare and exchange bytes - cmpxchg, - /// CPU identification - cpuid, - /// Convert doubleword to quadword - cqo, - /// Convert word to doubleword - cwd, - /// Convert word to doubleword - cwde, - /// Decimal adjust AL after addition - /// Decimal adjust AL after subtraction - da, - /// Decrement by 1 - /// Decrement stack-top pointer - /// Decrement shadow stack pointer - de, - /// Unsigned division - /// Signed division - /// Divide - /// Divide packed single-precision floating-point values - /// Divide scalar single-precision floating-point values - /// Divide packed double-precision floating-point values - /// Divide scalar double-precision floating-point values - div, - /// Terminate and indirect branch in 32-bit and compatibility mode - /// Terminate and indirect branch in 64-bit mode - endbr, - /// Enqueue command - /// Enqueue command supervisor - enqcmd, - /// Make stack frame for procedure parameters - /// Fast system call - enter, - /// Fast return from fast system call - exit, - /// Load fence - /// Memory fence - /// Store fence - fence, - /// Halt - hlt, - /// History reset - hreset, - /// Input from port - /// Input from port to string - /// Increment by 1 - /// Increment stack-top pointer - /// Increment shadow stack pointer - in, - /// Call to interrupt procedure - int, - /// Invalidate internal caches - /// Invalidate TLB entries - /// Invalidate process-context identifier - inv, - /// Conditional jump - /// Jump - j, - /// Load status flags into AH register - lahf, - /// Load access right byte - lar, - /// Load effective address - lea, - /// High level procedure exit - leave, - /// Load global descriptor table register - lgdt, - /// Load interrupt descriptor table register - lidt, - /// Load local descriptor table register - lldt, - /// Load machine status word - lmsw, - /// Load string - lod, - /// Loop according to ECX counter - loop, - /// Load segment limit - lsl, - /// Load task register - ltr, - /// Count the number of leading zero bits - lzcnt, - /// Move - /// Move data from string to string - /// Move data after swapping bytes - /// Move scalar single-precision floating-point value - /// Move scalar double-precision floating-point value - /// Move doubleword - /// Move quadword - /// Move aligned packed integer values - /// Move unaligned packed integer values - /// Move quadword from XMM to MMX technology register - /// Move quadword from MMX technology to XMM register - mov, - /// Move with sign extension - movsx, - /// Move with zero extension - movzx, - /// Multiply - /// Signed multiplication - /// Multiply packed single-precision floating-point values - /// Multiply scalar single-precision floating-point values - /// Multiply packed double-precision floating-point values - /// Multiply scalar double-precision floating-point values - /// Multiply packed unsigned doubleword integers - /// Multiply packed doubleword integers - /// Carry-less multiplication quadword - mul, - /// Two's complement negation - neg, - /// No-op - /// No operation - nop, - /// One's complement negation - not, - /// Logical or - /// Bitwise logical or of packed single-precision floating-point values - /// Bitwise logical or of packed double-precision floating-point values - @"or", - /// Output to port - /// Output string to port - out, - /// Spin loop hint - /// Timed pause - pause, - /// Pop - pop, - /// Return the count of number of bits set to 1 - popcnt, - /// Pop stack into EFLAGS register - popf, - /// Push - push, - /// Push EFLAGS register onto the stack - pushf, - /// Rotate left through carry - /// Rotate right through carry - rc, - /// Read FS segment base - /// Read GS segment base - /// Read from model specific register - /// Read processor ID - /// Read protection key rights for user pages - /// Read performance-monitoring counters - /// Read random number - /// Read random seed - /// Read shadow stack pointer - /// Read time-stamp counter - /// Read time-stamp counter and processor ID - rd, - /// Return - /// Return from fast system call - /// Interrupt return - /// User-interrupt return - ret, - /// Rotate left - /// Rotate right - /// Rotate right logical without affecting flags - ro, - /// Resume from system management mode - rsm, - /// Arithmetic shift left - /// Arithmetic shift right - /// Shift left arithmetic without affecting flags - sa, - /// Store AH into flags - sahf, - /// Integer subtraction with borrow - sbb, - /// Scan string - sca, - /// Send user interprocessor interrupt - senduipi, - /// Serialize instruction execution - serialize, - /// Set byte on condition - set, - /// Logical shift left - /// Double precision shift left - /// Logical shift right - /// Double precision shift right - /// Shift left logical without affecting flags - /// Shift right logical without affecting flags - sh, - /// Store interrupt descriptor table register - sidt, - /// Store local descriptor table register - sldt, - /// Store machine status word - smsw, - /// Subtract - /// Subtract packed integers - /// Subtract packed single-precision floating-point values - /// Subtract scalar single-precision floating-point values - /// Subtract packed double-precision floating-point values - /// Subtract scalar double-precision floating-point values - /// Packed single-precision floating-point horizontal subtract - /// Packed double-precision floating-point horizontal subtract - /// Packed horizontal subtract - /// Packed horizontal subtract and saturate - sub, - /// Set carry flag - /// Set direction flag - /// Set interrupt flag - /// Store binary coded decimal integer and pop - /// Store floating-point value - /// Store integer - /// Store x87 FPU control word - /// Store x87 FPU environment - /// Store x87 FPU status word - /// Store MXCSR register state - st, - /// Store string - sto, - /// Test condition - /// Logical compare - /// Packed bit test - @"test", - /// Undefined instruction - ud, - /// User level set up monitor address - umonitor, - /// Verify a segment for reading - /// Verify a segment for writing - ver, - /// Write to model specific register - /// Write to model specific register - /// Write to model specific register - /// Write to shadow stack - /// Write to user shadow stack - wr, - /// Exchange and add - xadd, - /// Exchange register/memory with register - /// Exchange register contents - xch, - /// Get value of extended control register - xgetbv, - /// Table look-up translation - xlat, - /// Logical exclusive-or - /// Bitwise logical xor of packed single-precision floating-point values - /// Bitwise logical xor of packed double-precision floating-point values - xor, - - // X87 - /// Compute 2^x-1 - @"2xm1", - /// Absolute value - abs, - /// Change sign - chs, - /// Clear exceptions - clex, - /// Compare floating-point values - com, - /// Compare floating-point values and set EFLAGS - /// Compare scalar ordered single-precision floating-point values - /// Compare scalar ordered double-precision floating-point values - comi, - /// Cosine - cos, - /// Reverse divide - divr, - /// Free floating-point register - free, - /// Initialize floating-point unit - init, - /// Load binary coded decimal integer - /// Load floating-point value - /// Load integer - /// Load constant - /// Load x87 FPU control word - /// Load x87 FPU environment - /// Load MXCSR register state - ld, - /// Partial arctangent - patan, - /// Partial remainder - prem, - /// Partial tangent - ptan, - /// Round to integer - rndint, - /// Restore x87 FPU state - /// Restore x87 FPU, MMX, XMM, and MXCSR state - rstor, - /// Store x87 FPU state - /// Save x87 FPU, MMX technology, and MXCSR state - save, - /// Scale - scale, - /// Sine - sin, - /// Sine and cosine - sincos, - /// Square root - /// Square root of packed single-precision floating-point values - /// Square root of scalar single-precision floating-point value - /// Square root of packed double-precision floating-point values - /// Square root of scalar double-precision floating-point value - sqrt, - /// Store integer with truncation - stt, - /// Reverse subtract - subr, - /// Test - tst, - /// Unordered compare floating-point values - ucom, - /// Unordered compare floating-point values and set EFLAGS - /// Unordered compare scalar single-precision floating-point values - /// Unordered compare scalar double-precision floating-point values - ucomi, - /// Wait - /// User level monitor wait - wait, - /// Examine floating-point - xam, - /// Extract exponent and significand - xtract, - /// Compute y * log2x - /// Compute y * log2(x + 1) - yl2x, - - // MMX - /// Pack with signed saturation - ackssw, - /// Pack with signed saturation - ackssd, - /// Pack with unsigned saturation - ackusw, - /// Add packed signed integers with signed saturation - adds, - /// Add packed unsigned integers with unsigned saturation - addus, - /// Logical and not - /// Bitwise logical and not of packed single-precision floating-point values - /// Bitwise logical and not of packed double-precision floating-point values - andn, - /// Compare packed data for equal - cmpeq, - /// Compare packed data for greater than - cmpgt, - /// Empty MMX technology state - emms, - /// Multiply and add packed signed and unsigned bytes - maddubs, - /// Multiply and add packed integers - maddw, - /// Multiply packed signed integers and store low result - mull, - /// Multiply packed signed integers and store high result - mulh, - /// Shift packed data left logical - sll, - /// Shift packed data right arithmetic - sra, - /// Shift packed data right logical - srl, - /// Subtract packed signed integers with signed saturation - subs, - /// Subtract packed unsigned integers with unsigned saturation - subus, - /// Unpack high data - unpckhbw, - /// Unpack high data - unpckhdq, - /// Unpack high data - unpckhwd, - /// Unpack low data - unpcklbw, - /// Unpack low data - unpckldq, - /// Unpack low data - unpcklwd, - - // SSE - /// Average packed integers - avg, - /// Convert packed doubleword integers to packed single-precision floating-point values - /// Convert packed doubleword integers to packed double-precision floating-point values - cvtpi2, - /// Convert packed single-precision floating-point values to packed doubleword integers - cvtps2pi, - /// Convert doubleword integer to scalar single-precision floating-point value - /// Convert doubleword integer to scalar double-precision floating-point value - cvtsi2, - /// Convert scalar single-precision floating-point value to doubleword integer - cvtss2si, - /// Convert with truncation packed single-precision floating-point values to packed doubleword integers - cvttps2pi, - /// Convert with truncation scalar single-precision floating-point value to doubleword integer - cvttss2si, - /// Extract byte - /// Extract word - /// Extract doubleword - /// Extract quadword - extr, - /// Insert byte - /// Insert word - /// Insert doubleword - /// Insert quadword - insr, - /// Maximum of packed single-precision floating-point values - /// Maximum of scalar single-precision floating-point values - /// Maximum of packed double-precision floating-point values - /// Maximum of scalar double-precision floating-point values - max, - /// Maximum of packed signed integers - maxs, - /// Maximum of packed unsigned integers - maxu, - /// Minimum of packed single-precision floating-point values - /// Minimum of scalar single-precision floating-point values - /// Minimum of packed double-precision floating-point values - /// Minimum of scalar double-precision floating-point values - min, - /// Minimum of packed signed integers - mins, - /// Minimum of packed unsigned integers - minu, - /// Move aligned packed single-precision floating-point values - /// Move aligned packed double-precision floating-point values - mova, - /// Move high packed single-precision floating-point values - /// Move high packed double-precision floating-point values - movh, - /// Move packed single-precision floating-point values high to low - movhl, - /// Move low packed single-precision floating-point values - /// Move low packed double-precision floating-point values - movl, - /// Move packed single-precision floating-point values low to high - movlh, - /// Move byte mask - /// Extract packed single precision floating-point sign mask - /// Extract packed double precision floating-point sign mask - movmsk, - /// Move unaligned packed single-precision floating-point values - /// Move unaligned packed double-precision floating-point values - movu, - /// Multiply packed unsigned integers and store high result - mulhu, - /// Prefetch data into caches - /// Prefetch data into caches with intent to write - prefetch, - /// Compute sum of absolute differences - sadb, - /// Packed interleave shuffle of quadruplets of single-precision floating-point values - /// Packed interleave shuffle of pairs of double-precision floating-point values - /// Shuffle packed doublewords - /// Shuffle packed words - shuf, - /// Unpack and interleave high packed single-precision floating-point values - /// Unpack and interleave high packed double-precision floating-point values - unpckh, - /// Unpack and interleave low packed single-precision floating-point values - /// Unpack and interleave low packed double-precision floating-point values - unpckl, - - // SSE2 - /// Convert packed doubleword integers to packed single-precision floating-point values - /// Convert packed doubleword integers to packed double-precision floating-point values - cvtdq2, - /// Convert packed double-precision floating-point values to packed doubleword integers - cvtpd2dq, - /// Convert packed double-precision floating-point values to packed doubleword integers - cvtpd2pi, - /// Convert packed double-precision floating-point values to packed single-precision floating-point values - cvtpd2, - /// Convert packed single-precision floating-point values to packed doubleword integers - cvtps2dq, - /// Convert packed single-precision floating-point values to packed double-precision floating-point values - cvtps2, - /// Convert scalar double-precision floating-point value to doubleword integer - cvtsd2si, - /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value - cvtsd2, - /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value - cvtss2, - /// Convert with truncation packed double-precision floating-point values to packed doubleword integers - cvttpd2dq, - /// Convert with truncation packed double-precision floating-point values to packed doubleword integers - cvttpd2pi, - /// Convert with truncation packed single-precision floating-point values to packed doubleword integers - cvttps2dq, - /// Convert with truncation scalar double-precision floating-point value to doubleword integer - cvttsd2si, - /// Galois field affine transformation inverse - gf2p8affineinvq, - /// Galois field affine transformation - gf2p8affineq, - /// Galois field multiply bytes - gf2p8mul, - /// Shuffle packed high words - shufh, - /// Shuffle packed low words - shufl, - /// Unpack high data - unpckhqdq, - /// Unpack low data - unpcklqdq, - - // SSE3 - /// Packed single-precision floating-point add/subtract - /// Packed double-precision floating-point add/subtract - addsub, - /// Replicate double floating-point values - movddup, - /// Replicate single floating-point values - movshdup, - /// Replicate single floating-point values - movsldup, - - // SSSE3 - /// Packed align right - alignr, - /// Packed multiply high with round and scale - mulhrs, - /// Packed sign - sign, - - // SSE4.1 - /// Pack with unsigned saturation - ackusd, - /// Blend packed single-precision floating-point values - /// Blend scalar single-precision floating-point values - /// Blend packed double-precision floating-point values - /// Blend scalar double-precision floating-point values - /// Blend packed dwords - blend, - /// Variable blend packed single-precision floating-point values - /// Variable blend scalar single-precision floating-point values - /// Variable blend packed double-precision floating-point values - /// Variable blend scalar double-precision floating-point values - blendv, - /// Dot product of packed single-precision floating-point values - /// Dot product of packed double-precision floating-point values - dp, - /// Extract packed floating-point values - /// Extract packed integer values - extract, - /// Insert scalar single-precision floating-point value - /// Insert packed floating-point values - insert, - /// Packed horizontal word minimum - minposu, - /// Packed move with sign extend - movsxb, - movsxd, - movsxw, - /// Packed move with zero extend - movzxb, - movzxd, - movzxw, - /// Round packed single-precision floating-point values - /// Round scalar single-precision floating-point value - /// Round packed double-precision floating-point values - /// Round scalar double-precision floating-point value - round, - - // SSE4.2 - /// Accumulate CRC32 value - crc32, - - // AES - /// Perform one round of an AES decryption flow - /// Perform ten rounds of AES decryption flow with key locker using 128-bit key - /// Perform ten rounds of AES decryption flow with key locker using 256-bit key - /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 128-bit key - /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 256-bit key - aesdec, - /// Perform last round of an AES decryption flow - aesdeclast, - /// Perform one round of an AES encryption flow - /// Perform ten rounds of AES encryption flow with key locker using 128-bit key - /// Perform ten rounds of AES encryption flow with key locker using 256-bit key - /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 128-bit key - /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 256-bit key - aesenc, - /// Perform last round of an AES encryption flow - aesenclast, - /// Perform the AES InvMixColumn transformation - aesimc, - /// AES round key generation assist - aeskeygenassist, - - // SHA - /// Perform four rounds of SHA1 operation - sha1rnds, - /// Calculate SHA1 state variable E after four rounds - sha1nexte, - /// Perform an intermediate calculation for the next four SHA1 message dwords - /// Perform a final calculation for the next four SHA1 message dwords - sha1msg, - /// Perform an intermediate calculation for the next four SHA256 message dwords - /// Perform a final calculation for the next four SHA256 message dwords - sha256msg, - /// Perform two rounds of SHA256 operation - sha256rnds, - - // AVX - /// Load with broadcast floating-point data - /// Load integer and broadcast - broadcast, - /// Conditional SIMD packed loads and stores - /// Condition SIMD integer packed loads and stores - maskmov, - /// Permute floating-point values - /// Permute integer values - perm2, - /// Permute in-lane pairs of double-precision floating-point values - /// Permute in-lane quadruples of single-precision floating-point values - permil, - - // BMI - /// Bit field extract - bextr, - /// Extract lowest set isolated bit - /// Get mask up to lowest set bit - /// Reset lowest set bit - bls, - /// Count the number of trailing zero bits - tzcnt, - - // BMI2 - /// Zero high bits starting with specified bit position - bzhi, - /// Parallel bits deposit - pdep, - /// Parallel bits extract - pext, - - // F16C - /// Convert 16-bit floating-point values to single-precision floating-point values - cvtph2, - /// Convert single-precision floating-point values to 16-bit floating-point values - cvtps2ph, - - // FMA - /// Fused multiply-add of packed single-precision floating-point values - /// Fused multiply-add of scalar single-precision floating-point values - /// Fused multiply-add of packed double-precision floating-point values - /// Fused multiply-add of scalar double-precision floating-point values - fmadd132, - /// Fused multiply-add of packed single-precision floating-point values - /// Fused multiply-add of scalar single-precision floating-point values - /// Fused multiply-add of packed double-precision floating-point values - /// Fused multiply-add of scalar double-precision floating-point values - fmadd213, - /// Fused multiply-add of packed single-precision floating-point values - /// Fused multiply-add of scalar single-precision floating-point values - /// Fused multiply-add of packed double-precision floating-point values - /// Fused multiply-add of scalar double-precision floating-point values - fmadd231, - - // AVX2 - /// Permute packed doubleword elements - /// Permute packed qword elements - /// Permute double-precision floating-point elements - /// Permute single-precision floating-point elements - perm, - /// Variable bit shift left logical - sllv, - /// Variable bit shift right arithmetic - srav, - /// Variable bit shift right logical - srlv, - - // ADX - /// Unsigned integer addition of two operands with overflow flag - ado, - - // AESKLE - /// Encode 128-bit key with key locker - /// Encode 256-bit key with key locker - encodekey, - /// Load internal wrapping key with key locker - loadiwkey, - - /// A pseudo instruction that requires special lowering. - /// This should be the only tag in this enum that doesn't - /// directly correspond to one or more instruction mnemonics. - pseudo, - }; - - pub const FixedTag = struct { Fixes, Tag }; - - pub const Ops = enum(u8) { - /// No data associated with this instruction (only mnemonic is used). - none, - /// Single register operand. - /// Uses `r` payload. - r, - /// Register, register operands. - /// Uses `rr` payload. - rr, - /// Register, register, register operands. - /// Uses `rrr` payload. - rrr, - /// Register, register, register, register operands. - /// Uses `rrrr` payload. - rrrr, - /// Register, register, register, immediate (byte) operands. - /// Uses `rrri` payload. - rrri, - /// Register, register, immediate (sign-extended) operands. - /// Uses `rri` payload. - rri_s, - /// Register, register, immediate (unsigned) operands. - /// Uses `rri` payload. - rri_u, - /// Register, immediate (sign-extended) operands. - /// Uses `ri` payload. - ri_s, - /// Register, immediate (unsigned) operands. - /// Uses `ri` payload. - ri_u, - /// Register, 64-bit unsigned immediate operands. - /// Uses `ri` payload with `i` index of extra data of type `Imm64`. - ri_64, - /// Immediate (sign-extended) operand. - /// Uses `i` payload. - i_s, - /// Immediate (unsigned) operand. - /// Uses `i` payload. - i_u, - /// Immediate (word), immediate (byte) operands. - /// Uses `ii` payload. - ii, - /// Immediate (byte), register operands. - /// Uses `ri` payload. - ir, - /// Register, memory operands. - /// Uses `rx` payload with extra data of type `Memory`. - rm, - /// Register, memory, register operands. - /// Uses `rrx` payload with extra data of type `Memory`. - rmr, - /// Register, memory, immediate (word) operands. - /// Uses `rix` payload with extra data of type `Memory`. - rmi, - /// Register, memory, immediate (signed) operands. - /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`. - rmi_s, - /// Register, memory, immediate (unsigned) operands. - /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`. - rmi_u, - /// Register, register, memory. - /// Uses `rrix` payload with extra data of type `Memory`. - rrm, - /// Register, register, memory, register. - /// Uses `rrrx` payload with extra data of type `Memory`. - rrmr, - /// Register, register, memory, immediate (byte) operands. - /// Uses `rrix` payload with extra data of type `Memory`. - rrmi, - /// Single memory operand. - /// Uses `x` payload with extra data of type `Memory`. - m, - /// Memory, immediate (sign-extend) operands. - /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`. - mi_s, - /// Memory, immediate (unsigned) operands. - /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`. - mi_u, - /// Memory, register operands. - /// Uses `rx` payload with extra data of type `Memory`. - mr, - /// Memory, register, register operands. - /// Uses `rrx` payload with extra data of type `Memory`. - mrr, - /// Memory, register, immediate (word) operands. - /// Uses `rix` payload with extra data of type `Memory`. - mri, - /// References another Mir instruction directly. - /// Uses `inst` payload. - inst, - /// References a nav. - /// Uses `nav` payload. - nav, - /// References an uav. - /// Uses `uav` payload. - uav, - /// References a lazy symbol. - /// Uses `lazy_sym` payload. - lazy_sym, - /// References an external symbol. - /// Uses `extern_func` payload. - extern_func, - - // Pseudo instructions: - - /// Conditional move if zero flag set and parity flag not set - /// Clobbers the source operand! - /// Uses `rr` payload. - pseudo_cmov_z_and_np_rr, - /// Conditional move if zero flag not set or parity flag set - /// Uses `rr` payload. - pseudo_cmov_nz_or_p_rr, - /// Conditional move if zero flag not set or parity flag set - /// Uses `rx` payload. - pseudo_cmov_nz_or_p_rm, - /// Set byte if zero flag set and parity flag not set - /// Requires a scratch register! - /// Uses `rr` payload. - pseudo_set_z_and_np_r, - /// Set byte if zero flag set and parity flag not set - /// Requires a scratch register! - /// Uses `rx` payload. - pseudo_set_z_and_np_m, - /// Set byte if zero flag not set or parity flag set - /// Requires a scratch register! - /// Uses `rr` payload. - pseudo_set_nz_or_p_r, - /// Set byte if zero flag not set or parity flag set - /// Requires a scratch register! - /// Uses `rx` payload. - pseudo_set_nz_or_p_m, - /// Jump if zero flag set and parity flag not set - /// Uses `inst` payload. - pseudo_j_z_and_np_inst, - /// Jump if zero flag not set or parity flag set - /// Uses `inst` payload. - pseudo_j_nz_or_p_inst, - - /// Probe alignment - /// Uses `ri` payload. - pseudo_probe_align_ri_s, - /// Probe adjust unrolled - /// Uses `ri` payload. - pseudo_probe_adjust_unrolled_ri_s, - /// Probe adjust setup - /// Uses `rri` payload. - pseudo_probe_adjust_setup_rri_s, - /// Probe adjust loop - /// Uses `rr` payload. - pseudo_probe_adjust_loop_rr, - - /// Push registers - /// Uses `reg_list` payload. - pseudo_push_reg_list, - /// Pop registers - /// Uses `reg_list` payload. - pseudo_pop_reg_list, - - /// Define cfa rule as offset from register. - /// Uses `ri` payload. - pseudo_cfi_def_cfa_ri_s, - /// Modify cfa rule register. - /// Uses `r` payload. - pseudo_cfi_def_cfa_register_r, - /// Modify cfa rule offset. - /// Uses `i` payload. - pseudo_cfi_def_cfa_offset_i_s, - /// Offset cfa rule offset. - /// Uses `i` payload. - pseudo_cfi_adjust_cfa_offset_i_s, - /// Define register rule as stored at offset from cfa. - /// Uses `ri` payload. - pseudo_cfi_offset_ri_s, - /// Define register rule as offset from cfa. - /// Uses `ri` payload. - pseudo_cfi_val_offset_ri_s, - /// Define register rule as stored at offset from cfa rule register. - /// Uses `ri` payload. - pseudo_cfi_rel_offset_ri_s, - /// Define register rule as register. - /// Uses `rr` payload. - pseudo_cfi_register_rr, - /// Define register rule from initial. - /// Uses `r` payload. - pseudo_cfi_restore_r, - /// Define register rule as undefined. - /// Uses `r` payload. - pseudo_cfi_undefined_r, - /// Define register rule as itself. - /// Uses `r` payload. - pseudo_cfi_same_value_r, - /// Push cfi state. - pseudo_cfi_remember_state_none, - /// Pop cfi state. - pseudo_cfi_restore_state_none, - /// Raw cfi bytes. - /// Uses `bytes` payload. - pseudo_cfi_escape_bytes, - - /// End of prologue - /// Uses `none` payload. - pseudo_dbg_prologue_end_none, - /// Update debug line with is_stmt register set - /// Uses `line_column` payload. - pseudo_dbg_line_stmt_line_column, - /// Update debug line with is_stmt register clear - /// Uses `line_column` payload. - pseudo_dbg_line_line_column, - /// Start of epilogue - /// Uses `none` payload. - pseudo_dbg_epilogue_begin_none, - /// Start of lexical block - /// Uses `none` payload. - pseudo_dbg_enter_block_none, - /// End of lexical block - /// Uses `none` payload. - pseudo_dbg_leave_block_none, - /// Start of inline function - /// Uses `ip_index` payload. - pseudo_dbg_enter_inline_func, - /// End of inline function - /// Uses `ip_index` payload. - pseudo_dbg_leave_inline_func, - /// Local argument. - /// Uses `none` payload. - pseudo_dbg_arg_none, - /// Local argument. - /// Uses `i` payload. - pseudo_dbg_arg_i_s, - /// Local argument. - /// Uses `i` payload. - pseudo_dbg_arg_i_u, - /// Local argument. - /// Uses `i64` payload. - pseudo_dbg_arg_i_64, - /// Local argument. - /// Uses `ro` payload. - pseudo_dbg_arg_ro, - /// Local argument. - /// Uses `fa` payload. - pseudo_dbg_arg_fa, - /// Local argument. - /// Uses `x` payload with extra data of type `Memory`. - pseudo_dbg_arg_m, - /// Local argument. - /// Uses `ip_index` payload. - pseudo_dbg_arg_val, - /// Remaining arguments are varargs. - pseudo_dbg_var_args_none, - /// Local variable. - /// Uses `none` payload. - pseudo_dbg_var_none, - /// Local variable. - /// Uses `i` payload. - pseudo_dbg_var_i_s, - /// Local variable. - /// Uses `i` payload. - pseudo_dbg_var_i_u, - /// Local variable. - /// Uses `i64` payload. - pseudo_dbg_var_i_64, - /// Local variable. - /// Uses `ro` payload. - pseudo_dbg_var_ro, - /// Local variable. - /// Uses `fa` payload. - pseudo_dbg_var_fa, - /// Local variable. - /// Uses `x` payload with extra data of type `Memory`. - pseudo_dbg_var_m, - /// Local variable. - /// Uses `ip_index` payload. - pseudo_dbg_var_val, - - /// Tombstone - /// Emitter should skip this instruction. - pseudo_dead_none, - }; - - pub const Data = union { - none: struct { - fixes: Fixes = ._, - }, - /// References another Mir instruction. - inst: struct { - fixes: Fixes = ._, - inst: Index, - }, - /// A 32-bit immediate value. - i64: u64, - i: struct { - fixes: Fixes = ._, - i: u32, - }, - ii: struct { - fixes: Fixes = ._, - i1: u16, - i2: u8, - }, - r: struct { - fixes: Fixes = ._, - r1: Register, - }, - rr: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - }, - rrr: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - r3: Register, - }, - rrrr: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - r3: Register, - r4: Register, - }, - rrri: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - r3: Register, - i: u8, - }, - rri: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - i: u32, - }, - /// Register, immediate. - ri: struct { - fixes: Fixes = ._, - r1: Register, - i: u32, - }, - /// Register, followed by custom payload found in extra. - rx: struct { - fixes: Fixes = ._, - r1: Register, - payload: u32, - }, - /// Register, register, followed by Custom payload found in extra. - rrx: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - payload: u32, - }, - /// Register, register, register, followed by Custom payload found in extra. - rrrx: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - r3: Register, - payload: u32, - }, - /// Register, byte immediate, followed by Custom payload found in extra. - rix: struct { - fixes: Fixes = ._, - r1: Register, - i: u16, - payload: u32, - }, - /// Register, register, byte immediate, followed by Custom payload found in extra. - rrix: struct { - fixes: Fixes = ._, - r1: Register, - r2: Register, - i: u8, - payload: u32, - }, - /// Custom payload found in extra. - x: struct { - fixes: Fixes = ._, - payload: u32, - }, - bytes: struct { - payload: u32, - len: u32, - - pub fn get(bytes: @This(), mir: Mir) []const u8 { - return std.mem.sliceAsBytes(mir.extra[bytes.payload..])[0..bytes.len]; - } - }, - fa: bits.FrameAddr, - ro: bits.RegisterOffset, - nav: bits.NavOffset, - uav: InternPool.Key.Ptr.BaseAddr.Uav, - lazy_sym: link.File.LazySymbol, - extern_func: Mir.NullTerminatedString, - /// Debug line and column position - line_column: struct { - line: u32, - column: u32, - }, - ip_index: InternPool.Index, - /// Register list - reg_list: RegisterList, - }; - - comptime { - if (!std.debug.runtime_safety) { - // Make sure we don't accidentally make instructions bigger than expected. - // Note that in safety builds, Zig is allowed to insert a secret field for safety checks. - assert(@sizeOf(Data) == 8); - } - const Mnemonic = @import("Encoding.zig").Mnemonic; - if (@typeInfo(Mnemonic).@"enum".fields.len != 977 or - @typeInfo(Fixes).@"enum".fields.len != 231 or - @typeInfo(Tag).@"enum".fields.len != 251) - { - const cond_src = (struct { - fn src() std.builtin.SourceLocation { - return @src(); - } - }).src(); - @setEvalBranchQuota(1_750_000); - for (@typeInfo(Mnemonic).@"enum".fields) |mnemonic| { - if (mnemonic.name[0] == '.') continue; - for (@typeInfo(Fixes).@"enum".fields) |fixes| { - const pattern = fixes.name[if (std.mem.indexOfScalar(u8, fixes.name, ' ')) |index| index + " ".len else 0..]; - const wildcard_index = std.mem.indexOfScalar(u8, pattern, '_').?; - const mnem_prefix = pattern[0..wildcard_index]; - const mnem_suffix = pattern[wildcard_index + "_".len ..]; - if (!std.mem.startsWith(u8, mnemonic.name, mnem_prefix)) continue; - if (!std.mem.endsWith(u8, mnemonic.name, mnem_suffix)) continue; - if (@hasField( - Tag, - mnemonic.name[mnem_prefix.len .. mnemonic.name.len - mnem_suffix.len], - )) break; - } else @compileError("'" ++ mnemonic.name ++ "' is not encodable in Mir"); - } - @compileError(std.fmt.comptimePrint( - \\All mnemonics are encodable in Mir! You may now change the condition at {s}:{d} to: - \\if (@typeInfo(Mnemonic).@"enum".fields.len != {d} or - \\ @typeInfo(Fixes).@"enum".fields.len != {d} or - \\ @typeInfo(Tag).@"enum".fields.len != {d}) - , .{ - cond_src.file, - cond_src.line - 6, - @typeInfo(Mnemonic).@"enum".fields.len, - @typeInfo(Fixes).@"enum".fields.len, - @typeInfo(Tag).@"enum".fields.len, - })); - } - } -}; - -/// Used in conjunction with payload to transfer a list of used registers in a compact manner. -pub const RegisterList = struct { - bitset: BitSet, - - const BitSet = std.bit_set.IntegerBitSet(32); - const Self = @This(); - - pub const empty: RegisterList = .{ .bitset = .initEmpty() }; - - fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt { - for (registers, 0..) |cpreg, i| { - if (reg.id() == cpreg.id()) return @intCast(i); - } - unreachable; // register not in input register list! - } - - pub fn push(self: *Self, registers: []const Register, reg: Register) void { - const index = getIndexForReg(registers, reg); - self.bitset.set(index); - } - - pub fn isSet(self: Self, registers: []const Register, reg: Register) bool { - const index = getIndexForReg(registers, reg); - return self.bitset.isSet(index); - } - - pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) { - return self.bitset.iterator(options); - } - - pub fn count(self: Self) i32 { - return @intCast(self.bitset.count()); - } - - pub fn size(self: Self, target: *const std.Target) i32 { - return @intCast(self.bitset.count() * @as(u4, switch (target.cpu.arch) { - else => unreachable, - .x86 => 4, - .x86_64 => 8, - })); - } -}; - -pub const NullTerminatedString = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn toSlice(nts: NullTerminatedString, mir: *const Mir) ?[:0]const u8 { - if (nts == .none) return null; - const string_bytes = mir.string_bytes[@intFromEnum(nts)..]; - return string_bytes[0..std.mem.indexOfScalar(u8, string_bytes, 0).? :0]; - } -}; - -pub const Local = struct { - name: NullTerminatedString, - type: InternPool.Index, -}; - -pub const Imm32 = struct { - imm: u32, -}; - -pub const Imm64 = struct { - msb: u32, - lsb: u32, - - pub fn encode(v: u64) Imm64 { - return .{ - .msb = @truncate(v >> 32), - .lsb = @truncate(v), - }; - } - - pub fn decode(imm: Imm64) u64 { - var res: u64 = 0; - res |= @as(u64, @intCast(imm.msb)) << 32; - res |= @as(u64, @intCast(imm.lsb)); - return res; - } -}; - -pub const Memory = struct { - info: Info, - base: u32, - off: u32, - extra: u32, - - pub const Info = packed struct(u32) { - base: @typeInfo(bits.Memory.Base).@"union".tag_type.?, - mod: @typeInfo(bits.Memory.Mod).@"union".tag_type.?, - size: bits.Memory.Size, - index: Register, - scale: bits.Memory.Scale, - _: u13 = undefined, - }; - - pub fn encode(mem: bits.Memory) Memory { - return .{ - .info = .{ - .base = mem.base, - .mod = mem.mod, - .size = switch (mem.mod) { - .rm => |rm| rm.size, - .off => undefined, - }, - .index = switch (mem.mod) { - .rm => |rm| rm.index, - .off => undefined, - }, - .scale = switch (mem.mod) { - .rm => |rm| rm.scale, - .off => undefined, - }, - }, - .base = switch (mem.base) { - .none, .table => undefined, - .reg => |reg| @intFromEnum(reg), - .frame => |frame_index| @intFromEnum(frame_index), - .rip_inst => |inst_index| inst_index, - .nav => |nav| @intFromEnum(nav), - .uav => |uav| @intFromEnum(uav.val), - .lazy_sym => |lazy_sym| @intFromEnum(lazy_sym.ty), - .extern_func => |extern_func| @intFromEnum(extern_func), - }, - .off = switch (mem.mod) { - .rm => |rm| @bitCast(rm.disp), - .off => |off| @truncate(off), - }, - .extra = switch (mem.mod) { - .rm => switch (mem.base) { - else => undefined, - .uav => |uav| @intFromEnum(uav.orig_ty), - .lazy_sym => |lazy_sym| @intFromEnum(lazy_sym.kind), - }, - .off => switch (mem.base) { - .reg => @intCast(mem.mod.off >> 32), - else => unreachable, - }, - }, - }; - } - - pub fn decode(mem: Memory) encoder.Instruction.Memory { - switch (mem.info.mod) { - .rm => { - if (mem.info.base == .reg and @as(Register, @enumFromInt(mem.base)) == .rip) { - assert(mem.info.index == .none and mem.info.scale == .@"1"); - return encoder.Instruction.Memory.initRip(mem.info.size, @bitCast(mem.off)); - } - return encoder.Instruction.Memory.initSib(mem.info.size, .{ - .disp = @bitCast(mem.off), - .base = switch (mem.info.base) { - .none => .none, - .reg => .{ .reg = @enumFromInt(mem.base) }, - .frame => .{ .frame = @enumFromInt(mem.base) }, - .table => .table, - .rip_inst => .{ .rip_inst = mem.base }, - .nav => .{ .nav = @enumFromInt(mem.base) }, - .uav => .{ .uav = .{ .val = @enumFromInt(mem.base), .orig_ty = @enumFromInt(mem.extra) } }, - .lazy_sym => .{ .lazy_sym = .{ .kind = @enumFromInt(mem.extra), .ty = @enumFromInt(mem.base) } }, - .extern_func => .{ .extern_func = @enumFromInt(mem.base) }, - }, - .scale_index = switch (mem.info.index) { - .none => null, - else => |index| .{ .scale = switch (mem.info.scale) { - inline else => |scale| comptime std.fmt.parseInt( - u4, - @tagName(scale), - 10, - ) catch unreachable, - }, .index = index }, - }, - }); - }, - .off => { - assert(mem.info.base == .reg); - return encoder.Instruction.Memory.initMoffs( - @enumFromInt(mem.base), - @as(u64, mem.extra) << 32 | mem.off, - ); - }, - } - } -}; - -pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { - mir.instructions.deinit(gpa); - gpa.free(mir.extra); - gpa.free(mir.string_bytes); - gpa.free(mir.locals); - gpa.free(mir.table); - gpa.free(mir.memoized_encodings); - mir.frame_locs.deinit(gpa); - mir.* = undefined; -} - -pub fn emit( - mir: Mir, - lf: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - func_index: InternPool.Index, - atom_index: u32, - w: *std.Io.Writer, - debug_output: link.File.DebugInfoOutput, -) codegen.CodeGenError!void { - const zcu = pt.zcu; - const comp = zcu.comp; - const gpa = comp.gpa; - const func = zcu.funcInfo(func_index); - const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?; - const nav = func.owner_nav; - const mod = zcu.navFileScope(nav).mod.?; - var e: Emit = .{ - .lower = .{ - .target = &mod.resolved_target.result, - .allocator = gpa, - .mir = mir, - .cc = fn_info.cc, - .src_loc = src_loc, - }, - .bin_file = lf, - .pt = pt, - .pic = mod.pic, - .atom_index = atom_index, - .debug_output = debug_output, - .w = w, - - .prev_di_loc = .{ - .line = func.lbrace_line, - .column = func.lbrace_column, - .is_stmt = switch (debug_output) { - .dwarf => |dwarf| dwarf.dwarf.debug_line.header.default_is_stmt, - .none => undefined, - }, - }, - .prev_di_pc = 0, - - .code_offset_mapping = .empty, - .relocs = .empty, - .table_relocs = .empty, - }; - defer e.deinit(); - e.emitMir() catch |err| switch (err) { - error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, e.lower.err_msg.?), - error.InvalidInstruction, error.CannotEncode => return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}), - else => return zcu.codegenFail(nav, "emit MIR failed: {s}", .{@errorName(err)}), - }; -} - -pub fn emitLazy( - mir: Mir, - lf: *link.File, - pt: Zcu.PerThread, - src_loc: Zcu.LazySrcLoc, - lazy_sym: link.File.LazySymbol, - atom_index: u32, - w: *std.Io.Writer, - debug_output: link.File.DebugInfoOutput, -) codegen.CodeGenError!void { - const zcu = pt.zcu; - const comp = zcu.comp; - const gpa = comp.gpa; - const mod = comp.root_mod; - var e: Emit = .{ - .lower = .{ - .target = &mod.resolved_target.result, - .allocator = gpa, - .mir = mir, - .cc = .auto, - .src_loc = src_loc, - }, - .bin_file = lf, - .pt = pt, - .pic = mod.pic, - .atom_index = atom_index, - .debug_output = debug_output, - .w = w, - - .prev_di_loc = undefined, - .prev_di_pc = undefined, - - .code_offset_mapping = .empty, - .relocs = .empty, - .table_relocs = .empty, - }; - defer e.deinit(); - e.emitMir() catch |err| switch (err) { - error.LowerFail, error.EmitFail => return zcu.codegenFailTypeMsg(lazy_sym.ty, e.lower.err_msg.?), - error.InvalidInstruction, error.CannotEncode => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}), - else => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s}", .{@errorName(err)}), - }; -} - -pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end: u32 } { - const fields = std.meta.fields(T); - var i: u32 = index; - var result: T = undefined; - inline for (fields) |field| { - @field(result, field.name) = switch (field.type) { - u32 => mir.extra[i], - i32, Memory.Info => @bitCast(mir.extra[i]), - bits.FrameIndex => @enumFromInt(mir.extra[i]), - else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)), - }; - i += 1; - } - return .{ - .data = result, - .end = i, - }; -} - -pub const FrameLoc = struct { - base: Register, - disp: i32, -}; - -pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffset { - const frame_loc = mir.frame_locs.get(@intFromEnum(frame_addr.index)); - return .{ .reg = frame_loc.base, .off = frame_loc.disp + frame_addr.off }; -} - -pub fn resolveMemoryExtra(mir: Mir, payload: u32) Memory { - const mem = mir.extraData(Mir.Memory, payload).data; - return switch (mem.info.base) { - .none, .reg, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => mem, - .frame => if (mir.frame_locs.len > 0) .{ - .info = .{ - .base = .reg, - .mod = mem.info.mod, - .size = mem.info.size, - .index = mem.info.index, - .scale = mem.info.scale, - }, - .base = @intFromEnum(mir.frame_locs.items(.base)[mem.base]), - .off = @bitCast(mir.frame_locs.items(.disp)[mem.base] + @as(i32, @bitCast(mem.off))), - .extra = mem.extra, - } else mem, - }; -} - -const assert = std.debug.assert; -const bits = @import("bits.zig"); -const builtin = @import("builtin"); -const encoder = @import("encoder.zig"); -const std = @import("std"); - -const InternPool = @import("../../InternPool.zig"); -const Mir = @This(); -const Register = bits.Register; -const Emit = @import("Emit.zig"); -const codegen = @import("../../codegen.zig"); -const link = @import("../../link.zig"); -const Zcu = @import("../../Zcu.zig"); diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig deleted file mode 100644 index 041eb1632d3b91bb5a9e47d8a1d800c2682f6b6b..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/abi.zig +++ /dev/null @@ -1,608 +0,0 @@ -pub const Class = enum { - /// INTEGER: This class consists of integral types that fit into one of the general - /// purpose registers. - integer, - /// SSE: The class consists of types that fit into a vector register. - sse, - /// SSEUP: The class consists of types that fit into a vector register and can be passed - /// and returned in the upper bytes of it. - sseup, - /// X87, X87UP: These classes consist of types that will be returned via the - /// x87 FPU. - x87, - /// The 15-bit exponent, 1-bit sign, and 6 bytes of padding of an `f80`. - x87up, - /// NO_CLASS: This class is used as initializer in the algorithms. It will be used for - /// padding and empty structures and unions. - none, - /// MEMORY: This class consists of types that will be passed and returned in mem- - /// ory via the stack. - memory, - /// Win64 passes 128-bit integers as `Class.memory` but returns them as `Class.sse`. - win_i128, - /// A `Class.sse` containing one `f32`. - float, - /// A `Class.sse` containing two `f32`s. - float_combine, - /// Clang passes each vector element in a separate `Class.integer`, but returns as `Class.memory`. - integer_per_element, - - pub const one_integer: [8]Class = .{ - .integer, .none, .none, .none, - .none, .none, .none, .none, - }; - pub const two_integers: [8]Class = .{ - .integer, .integer, .none, .none, - .none, .none, .none, .none, - }; - pub const three_integers: [8]Class = .{ - .integer, .integer, .integer, .none, - .none, .none, .none, .none, - }; - pub const four_integers: [8]Class = .{ - .integer, .integer, .integer, .integer, - .none, .none, .none, .none, - }; - pub const len_integers: [8]Class = .{ - .integer_per_element, .none, .none, .none, - .none, .none, .none, .none, - }; - - pub const @"f16" = @"f64"; - pub const @"f32": [8]Class = .{ - .float, .none, .none, .none, - .none, .none, .none, .none, - }; - pub const @"f64": [8]Class = .{ - .sse, .none, .none, .none, - .none, .none, .none, .none, - }; - pub const @"f80": [8]Class = .{ - .x87, .x87up, .none, .none, - .none, .none, .none, .none, - }; - pub const @"f128": [8]Class = .{ - .sse, .sseup, .none, .none, - .none, .none, .none, .none, - }; - - /// COMPLEX_X87: This class consists of types that will be returned via the x87 - /// FPU. - pub const complex_x87: [8]Class = .{ - .x87, .x87up, .x87, .x87up, - .none, .none, .none, .none, - }; - - pub const stack: [8]Class = .{ - .memory, .none, .none, .none, - .none, .none, .none, .none, - }; - - pub fn isX87(class: Class) bool { - return switch (class) { - .x87, .x87up => true, - else => false, - }; - } - - /// Combine a field class with the prev one. - fn combineSystemV(prev_class: Class, next_class: Class) Class { - // "If both classes are equal, this is the resulting class." - if (prev_class == next_class) - return if (prev_class == .float) .float_combine else prev_class; - - // "If one of the classes is NO_CLASS, the resulting class - // is the other class." - if (prev_class == .none) return next_class; - - // "If one of the classes is MEMORY, the result is the MEMORY class." - if (prev_class == .memory or next_class == .memory) return .memory; - - // "If one of the classes is INTEGER, the result is the INTEGER." - if (prev_class == .integer or next_class == .integer) return .integer; - - // "If one of the classes is X87, X87UP, COMPLEX_X87 class, - // MEMORY is used as class." - if (prev_class.isX87() or next_class.isX87()) return .memory; - - // "Otherwise class SSE is used." - return .sse; - } -}; - -pub fn classifyWindows(ty: Type, zcu: *Zcu, target: *const std.Target) Class { - // https://docs.microsoft.com/en-gb/cpp/build/x64-calling-convention?view=vs-2017 - // "There's a strict one-to-one correspondence between a function call's arguments - // and the registers used for those arguments. Any argument that doesn't fit in 8 - // bytes, or isn't 1, 2, 4, or 8 bytes, must be passed by reference. A single argument - // is never spread across multiple registers." - // "All floating point operations are done using the 16 XMM registers." - // "Structs and unions of size 8, 16, 32, or 64 bits, and __m64 types, are passed - // as if they were integers of the same size." - return switch (ty.zigTypeTag(zcu)) { - .pointer, - .int, - .bool, - .@"enum", - .void, - .noreturn, - .error_set, - .@"struct", - .@"union", - .optional, - .array, - .error_union, - .@"anyframe", - .frame, - => switch (ty.abiSize(zcu)) { - 0 => unreachable, - 1, 2, 4, 8 => .integer, - else => switch (ty.zigTypeTag(zcu)) { - .int => .win_i128, - .@"struct", .@"union" => if (ty.containerLayout(zcu) == .@"packed") - .win_i128 - else - .memory, - else => .memory, - }, - }, - - .float => switch (ty.floatBits(target)) { - 16, 32, 64, 128 => .sse, - 80 => .memory, - else => unreachable, - }, - .vector => .sse, - - .type, - .comptime_float, - .comptime_int, - .undefined, - .null, - .@"fn", - .@"opaque", - .enum_literal, - => unreachable, - }; -} - -pub const Context = enum { ret, arg, other }; - -/// There are a maximum of 8 possible return slots. Returned values are in -/// the beginning of the array; unused slots are filled with .none. -pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Context) [8]Class { - switch (ty.zigTypeTag(zcu)) { - .pointer => switch (ty.ptrSize(zcu)) { - .slice => return Class.two_integers, - else => return Class.one_integer, - }, - .int, .@"enum", .error_set => { - const bits = ty.intInfo(zcu).bits; - if (bits <= 64 * 1) return Class.one_integer; - if (bits <= 64 * 2) return Class.two_integers; - if (bits <= 64 * 3) return Class.three_integers; - if (bits <= 64 * 4) return Class.four_integers; - return Class.stack; - }, - .bool, .void, .noreturn => return Class.one_integer, - .float => switch (ty.floatBits(target)) { - 16 => { - if (ctx == .other) return Class.stack; - // TODO clang doesn't allow __fp16 as .ret or .arg - return Class.f16; - }, - 32 => return Class.f32, - 64 => return Class.f64, - // "Arguments of types __float128, _Decimal128 and __m128 are - // split into two halves. The least significant ones belong - // to class SSE, the most significant one to class SSEUP." - 128 => return Class.f128, - // "The 64-bit mantissa of arguments of type long double - // belongs to class X87, the 16-bit exponent plus 6 bytes - // of padding belongs to class X87UP." - 80 => return Class.f80, - else => unreachable, - }, - .vector => { - const elem_ty = ty.childType(zcu); - const bits = elem_ty.bitSize(zcu) * ty.arrayLen(zcu); - if (elem_ty.toIntern() == .bool_type) { - if (bits <= 32) return Class.one_integer; - if (bits <= 64) return Class.f64; - if (ctx == .other) return Class.stack; - if (bits <= 128) return Class.len_integers; - if (bits <= 256 and target.cpu.has(.x86, .avx)) return Class.len_integers; - if (bits <= 512 and target.cpu.has(.x86, .avx512f)) return Class.len_integers; - return Class.stack; - } - if (elem_ty.isRuntimeFloat() and elem_ty.floatBits(target) == 80) { - if (bits <= 80 * 1) return Class.f80; - if (bits <= 80 * 2) return Class.complex_x87; - return Class.stack; - } - if (bits <= 64 * 1) return .{ - .sse, .none, .none, .none, - .none, .none, .none, .none, - }; - if (bits <= 64 * 2) return .{ - .sse, .sseup, .none, .none, - .none, .none, .none, .none, - }; - if (ctx == .arg and !target.cpu.has(.x86, .avx)) return Class.stack; - if (bits <= 64 * 3) return .{ - .sse, .sseup, .sseup, .none, - .none, .none, .none, .none, - }; - if (bits <= 64 * 4) return .{ - .sse, .sseup, .sseup, .sseup, - .none, .none, .none, .none, - }; - if (ctx == .arg and !target.cpu.has(.x86, .avx512f)) return Class.stack; - if (bits <= 64 * 5) return .{ - .sse, .sseup, .sseup, .sseup, - .sseup, .none, .none, .none, - }; - if (bits <= 64 * 6) return .{ - .sse, .sseup, .sseup, .sseup, - .sseup, .sseup, .none, .none, - }; - if (bits <= 64 * 7) return .{ - .sse, .sseup, .sseup, .sseup, - .sseup, .sseup, .sseup, .none, - }; - if (bits <= 64 * 8 or (ctx == .ret and bits <= @as(u64, if (target.cpu.has(.x86, .avx512f)) - 64 * 32 - else if (target.cpu.has(.x86, .avx)) - 64 * 16 - else - 64 * 8))) return .{ - .sse, .sseup, .sseup, .sseup, - .sseup, .sseup, .sseup, .sseup, - }; - return Class.stack; - }, - .optional => { - if (ty.optionalReprIsPayload(zcu)) { - return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx); - } - return Class.stack; - }, - .@"struct", .@"union" => { - // "If the size of an object is larger than eight eightbytes, or - // it contains unaligned fields, it has class MEMORY" - // "If the size of the aggregate exceeds a single eightbyte, each is classified - // separately.". - const ty_size = ty.abiSize(zcu); - switch (ty.containerLayout(zcu)) { - .auto => unreachable, - .@"extern" => {}, - .@"packed" => { - if (ty_size <= 8) return Class.one_integer; - if (ty_size <= 16) return Class.two_integers; - unreachable; // frontend should not have allowed this type as extern - }, - } - if (ty_size > 64) return Class.stack; - - var result: [8]Class = @splat(.none); - _ = if (zcu.typeToStruct(ty)) |loaded_struct| - classifySystemVStruct(&result, 0, loaded_struct, zcu, target) - else if (zcu.typeToUnion(ty)) |loaded_union| - classifySystemVUnion(&result, 0, loaded_union, zcu, target) - else - unreachable; - - // Post-merger cleanup - - // "If one of the classes is MEMORY, the whole argument is passed in memory" - // "If X87UP is not preceded by X87, the whole argument is passed in memory." - for (result, 0..) |class, i| switch (class) { - .memory => return Class.stack, - .x87up => if (i == 0 or result[i - 1] != .x87) return Class.stack, - else => continue, - }; - // "If the size of the aggregate exceeds two eightbytes and the first eight- - // byte isn’t SSE or any other eightbyte isn’t SSEUP, the whole argument - // is passed in memory." - if (ty_size > 16 and (result[0] != .sse or - std.mem.indexOfNone(Class, result[1..], &.{ .sseup, .none }) != null)) return Class.stack; - - // "If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE." - for (&result, 0..) |*item, i| { - if (item.* == .sseup) switch (result[i - 1]) { - .sse, .sseup => continue, - else => item.* = .sse, - }; - } - return result; - }, - .array => { - const ty_size = ty.abiSize(zcu); - if (ty_size <= 8) return Class.one_integer; - if (ty_size <= 16) return Class.two_integers; - return Class.stack; - }, - else => unreachable, - } -} - -fn classifySystemVStruct( - result: *[8]Class, - starting_byte_offset: u64, - loaded_struct: InternPool.LoadedStructType, - zcu: *Zcu, - target: *const std.Target, -) u64 { - const ip = &zcu.intern_pool; - var byte_offset = starting_byte_offset; - var field_it = loaded_struct.iterateRuntimeOrder(ip); - while (field_it.next()) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); - const field_align = loaded_struct.fieldAlign(ip, field_index); - byte_offset = std.mem.alignForward( - u64, - byte_offset, - field_align.toByteUnits() orelse field_ty.abiAlignment(zcu).toByteUnits().?, - ); - if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { - switch (field_loaded_struct.layout) { - .auto => unreachable, - .@"extern" => { - byte_offset = classifySystemVStruct(result, byte_offset, field_loaded_struct, zcu, target); - continue; - }, - .@"packed" => {}, - } - } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| { - switch (field_loaded_union.flagsUnordered(ip).layout) { - .auto => unreachable, - .@"extern" => { - byte_offset = classifySystemVUnion(result, byte_offset, field_loaded_union, zcu, target); - continue; - }, - .@"packed" => {}, - } - } - const field_classes = std.mem.sliceTo(&classifySystemV(field_ty, zcu, target, .other), .none); - for (result[@intCast(byte_offset / 8)..][0..field_classes.len], field_classes) |*result_class, field_class| - result_class.* = result_class.combineSystemV(field_class); - byte_offset += field_ty.abiSize(zcu); - } - const final_byte_offset = starting_byte_offset + loaded_struct.sizeUnordered(ip); - std.debug.assert(final_byte_offset == std.mem.alignForward( - u64, - byte_offset, - loaded_struct.flagsUnordered(ip).alignment.toByteUnits().?, - )); - return final_byte_offset; -} - -fn classifySystemVUnion( - result: *[8]Class, - starting_byte_offset: u64, - loaded_union: InternPool.LoadedUnionType, - zcu: *Zcu, - target: *const std.Target, -) u64 { - const ip = &zcu.intern_pool; - for (0..loaded_union.field_types.len) |field_index| { - const field_ty = Type.fromInterned(loaded_union.field_types.get(ip)[field_index]); - if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { - switch (field_loaded_struct.layout) { - .auto => unreachable, - .@"extern" => { - _ = classifySystemVStruct(result, starting_byte_offset, field_loaded_struct, zcu, target); - continue; - }, - .@"packed" => {}, - } - } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| { - switch (field_loaded_union.flagsUnordered(ip).layout) { - .auto => unreachable, - .@"extern" => { - _ = classifySystemVUnion(result, starting_byte_offset, field_loaded_union, zcu, target); - continue; - }, - .@"packed" => {}, - } - } - const field_classes = std.mem.sliceTo(&classifySystemV(field_ty, zcu, target, .other), .none); - for (result[@intCast(starting_byte_offset / 8)..][0..field_classes.len], field_classes) |*result_class, field_class| - result_class.* = result_class.combineSystemV(field_class); - } - return starting_byte_offset + loaded_union.sizeUnordered(ip); -} - -pub const zigcc = struct { - pub const stack_align: ?InternPool.Alignment = null; - pub const return_in_regs = true; - pub const params_in_regs = true; - - const volatile_gpr = gp_regs.len - 5; - const volatile_x87 = x87_regs.len - 1; - const volatile_sse = sse_avx_regs.len; - - /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them - /// for anything else but stack offset tracking therefore we exclude them from this set. - pub const callee_preserved_regs = gp_regs[volatile_gpr..] ++ x87_regs[volatile_x87 .. x87_regs.len - 1] ++ sse_avx_regs[volatile_sse..]; - /// These registers need to be preserved (saved on the stack) and restored by the caller before - /// the caller relinquishes control to a subroutine via call instruction (or similar). - /// In other words, these registers are free to use by the callee. - pub const caller_preserved_regs = gp_regs[0..volatile_gpr] ++ x87_regs[0..volatile_x87] ++ sse_avx_regs[0..volatile_sse]; - - const int_param_regs = gp_regs[0 .. volatile_gpr - 1]; - const x87_param_regs = x87_regs[0..volatile_x87]; - const sse_param_regs = sse_avx_regs[0 .. volatile_sse / 2]; - const int_return_regs = gp_regs[0..volatile_gpr]; - const x87_return_regs = x87_regs[0..volatile_x87]; - const sse_return_regs = sse_avx_regs[0..volatile_gpr]; -}; - -pub const SysV = struct { - /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them - /// for anything else but stack offset tracking therefore we exclude them from this set. - pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 }; - /// These registers need to be preserved (saved on the stack) and restored by the caller before - /// the caller relinquishes control to a subroutine via call instruction (or similar). - /// In other words, these registers are free to use by the callee. - pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs; - - pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; - pub const c_abi_x87_param_regs = x87_regs[0..0]; - pub const c_abi_sse_param_regs = sse_avx_regs[0..8]; - pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; - pub const c_abi_x87_return_regs = x87_regs[0..2]; - pub const c_abi_sse_return_regs = sse_avx_regs[0..4]; -}; - -pub const Win64 = struct { - /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them - /// for anything else but stack offset tracking therefore we exclude them from this set. - pub const callee_preserved_regs = [_]Register{ .rbx, .rsi, .rdi, .r12, .r13, .r14, .r15 }; - /// These registers need to be preserved (saved on the stack) and restored by the caller before - /// the caller relinquishes control to a subroutine via call instruction (or similar). - /// In other words, these registers are free to use by the callee. - pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs; - - pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 }; - pub const c_abi_x87_param_regs = x87_regs[0..0]; - pub const c_abi_sse_param_regs = sse_avx_regs[0..4]; - pub const c_abi_int_return_regs = [_]Register{.rax}; - pub const c_abi_x87_return_regs = x87_regs[0..0]; - pub const c_abi_sse_return_regs = sse_avx_regs[0..1]; -}; - -pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.callee_preserved_regs, - .x86_64_sysv => &SysV.callee_preserved_regs, - .x86_64_win => &Win64.callee_preserved_regs, - else => unreachable, - }; -} - -pub fn getCallerPreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.caller_preserved_regs, - .x86_64_sysv => &SysV.caller_preserved_regs, - .x86_64_win => &Win64.caller_preserved_regs, - else => unreachable, - }; -} - -pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.int_param_regs, - .x86_64_sysv => &SysV.c_abi_int_param_regs, - .x86_64_win => &Win64.c_abi_int_param_regs, - else => unreachable, - }; -} - -pub fn getCAbiX87ParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.x87_param_regs, - .x86_64_sysv => SysV.c_abi_x87_param_regs, - .x86_64_win => Win64.c_abi_x87_param_regs, - else => unreachable, - }; -} - -pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention.Tag, target: *const std.Target) []const Register { - return switch (cc) { - .auto => switch (target.cpu.arch) { - else => unreachable, - .x86 => zigcc.sse_param_regs[0 .. zigcc.sse_param_regs.len / 2], - .x86_64 => zigcc.sse_param_regs, - }, - .x86_64_sysv => SysV.c_abi_sse_param_regs, - .x86_64_win => Win64.c_abi_sse_param_regs, - else => unreachable, - }; -} - -pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.int_return_regs, - .x86_64_sysv => &SysV.c_abi_int_return_regs, - .x86_64_win => &Win64.c_abi_int_return_regs, - else => unreachable, - }; -} - -pub fn getCAbiX87ReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.x87_return_regs, - .x86_64_sysv => SysV.c_abi_x87_return_regs, - .x86_64_win => Win64.c_abi_x87_return_regs, - else => unreachable, - }; -} - -pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register { - return switch (cc) { - .auto => zigcc.sse_return_regs, - .x86_64_sysv => SysV.c_abi_sse_return_regs, - .x86_64_win => Win64.c_abi_sse_return_regs, - else => unreachable, - }; -} - -pub fn getCAbiLinkerScratchReg(cc: std.builtin.CallingConvention.Tag) Register { - return switch (cc) { - .auto => zigcc.int_return_regs[zigcc.int_return_regs.len - 1], - .x86_64_sysv => SysV.c_abi_int_return_regs[0], - .x86_64_win => Win64.c_abi_int_return_regs[0], - else => unreachable, - }; -} - -const gp_regs = [_]Register{ - .rax, .rdx, .rbx, .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15, -}; -const x87_regs = [_]Register{ - .st0, .st1, .st2, .st3, .st4, .st5, .st6, .st7, -}; -const sse_avx_regs = [_]Register{ - .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, - .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, -}; -const allocatable_regs = gp_regs ++ x87_regs[0 .. x87_regs.len - 1] ++ sse_avx_regs; -pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, allocatable_regs); - -// Register classes -const RegisterBitSet = RegisterManager.RegisterBitSet; -pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.general_purpose)) set.set(index); - break :blk set; - }; - pub const gphi: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.gphi)) set.set(index); - break :blk set; - }; - pub const x87: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.x87)) set.set(index); - break :blk set; - }; - pub const sse: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.sse)) set.set(index); - break :blk set; - }; -}; - -const builtin = @import("builtin"); -const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; - -const InternPool = @import("../../InternPool.zig"); -const Register = @import("bits.zig").Register; -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const Type = @import("../../Type.zig"); -const Value = @import("../../Value.zig"); -const Zcu = @import("../../Zcu.zig"); diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig deleted file mode 100644 index 05048109335e0256d91c5f350d7fa900ce08e402..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/bits.zig +++ /dev/null @@ -1,912 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const expect = std.testing.expect; - -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; -const InternPool = @import("../../InternPool.zig"); -const link = @import("../../link.zig"); -const Mir = @import("Mir.zig"); - -/// EFLAGS condition codes -pub const Condition = enum(u5) { - /// above - a, - /// above or equal - ae, - /// below - b, - /// below or equal - be, - /// carry - c, - /// equal - e, - /// greater - g, - /// greater or equal - ge, - /// less - l, - /// less or equal - le, - /// not above - na, - /// not above or equal - nae, - /// not below - nb, - /// not below or equal - nbe, - /// not carry - nc, - /// not equal - ne, - /// not greater - ng, - /// not greater or equal - nge, - /// not less - nl, - /// not less or equal - nle, - /// not overflow - no, - /// not parity - np, - /// not sign - ns, - /// not zero - nz, - /// overflow - o, - /// parity - p, - /// parity even - pe, - /// parity odd - po, - /// sign - s, - /// zero - z, - - // Pseudo conditions - /// zero and not parity - z_and_np, - /// not zero or parity - nz_or_p, - - /// Converts a std.math.CompareOperator into a condition flag, - /// i.e. returns the condition that is true iff the result of the - /// comparison is true. Assumes signed comparison - pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) Condition { - return switch (op) { - .gte => .ge, - .gt => .g, - .neq => .ne, - .lt => .l, - .lte => .le, - .eq => .e, - }; - } - - /// Converts a std.math.CompareOperator into a condition flag, - /// i.e. returns the condition that is true iff the result of the - /// comparison is true. Assumes unsigned comparison - pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) Condition { - return switch (op) { - .gte => .ae, - .gt => .a, - .neq => .ne, - .lt => .b, - .lte => .be, - .eq => .e, - }; - } - - pub fn fromCompareOperator( - signedness: std.builtin.Signedness, - op: std.math.CompareOperator, - ) Condition { - return switch (signedness) { - .signed => fromCompareOperatorSigned(op), - .unsigned => fromCompareOperatorUnsigned(op), - }; - } - - /// Returns the condition which is true iff the given condition is false - pub fn negate(cond: Condition) Condition { - return switch (cond) { - .a => .na, - .ae => .nae, - .b => .nb, - .be => .nbe, - .c => .nc, - .e => .ne, - .g => .ng, - .ge => .nge, - .l => .nl, - .le => .nle, - .na => .a, - .nae => .ae, - .nb => .b, - .nbe => .be, - .nc => .c, - .ne => .e, - .ng => .g, - .nge => .ge, - .nl => .l, - .nle => .le, - .no => .o, - .np => .p, - .ns => .s, - .nz => .z, - .o => .no, - .p => .np, - .pe => .po, - .po => .pe, - .s => .ns, - .z => .nz, - - .z_and_np => .nz_or_p, - .nz_or_p => .z_and_np, - }; - } - - /// Returns the equivalent condition when the operands are swapped. - pub fn commute(cond: Condition) Condition { - return switch (cond) { - else => cond, - .a => .b, - .ae => .be, - .b => .a, - .be => .ae, - .g => .l, - .ge => .le, - .l => .g, - .le => .ge, - .na => .nb, - .nae => .nbe, - .nb => .na, - .nbe => .nae, - .ng => .nl, - .nge => .nle, - .nl => .ng, - .nle => .nge, - }; - } -}; - -/// The immediate operand of vcvtps2ph. -pub const RoundMode = packed struct(u5) { - direction: Direction = .mxcsr, - precision: enum(u1) { - normal = 0b0, - inexact = 0b1, - } = .normal, - - pub const Direction = enum(u4) { - /// Round to nearest (even) - nearest = 0b0_00, - /// Round down (toward -∞) - down = 0b0_01, - /// Round up (toward +∞) - up = 0b0_10, - /// Round toward zero (truncate) - zero = 0b0_11, - /// Use current rounding mode of MXCSR.RC - mxcsr = 0b1_00, - }; - - pub fn imm(mode: RoundMode) Immediate { - return .u(@as(@typeInfo(RoundMode).@"struct".backing_integer.?, @bitCast(mode))); - } -}; - -/// The immediate operand of cmppd, cmpps, cmpsd, and cmpss. -pub const SseFloatPredicate = enum(u3) { - /// Equal (ordered, non-signaling) - eq, - /// Less-than (ordered, signaling) - lt, - /// Less-than-or-equal (ordered, signaling) - le, - /// Unordered (non-signaling) - unord, - /// Not-equal (unordered, non-signaling) - neq, - /// Not-less-than (unordered, signaling) - nlt, - /// Not-less-than-or-equal (unordered, signaling) - nle, - /// Ordered (non-signaling) - ord, - - /// Equal (ordered, non-signaling) - pub const eq_oq: SseFloatPredicate = .eq; - /// Less-than (ordered, signaling) - pub const lt_os: SseFloatPredicate = .lt; - /// Less-than-or-equal (ordered, signaling) - pub const le_os: SseFloatPredicate = .le; - /// Unordered (non-signaling) - pub const unord_q: SseFloatPredicate = .unord; - /// Not-equal (unordered, non-signaling) - pub const neq_uq: SseFloatPredicate = .neq; - /// Not-less-than (unordered, signaling) - pub const nlt_us: SseFloatPredicate = .nlt; - /// Not-less-than-or-equal (unordered, signaling) - pub const nle_us: SseFloatPredicate = .nle; - /// Ordered (non-signaling) - pub const ord_q: SseFloatPredicate = .ord; - - pub fn imm(pred: SseFloatPredicate) Immediate { - return .u(@intFromEnum(pred)); - } -}; - -/// The immediate operand of vcmppd, vcmpps, vcmpsd, and vcmpss. -pub const VexFloatPredicate = enum(u5) { - /// Equal (ordered, non-signaling) - eq_oq, - /// Less-than (ordered, signaling) - lt_os, - /// Less-than-or-equal (ordered, signaling) - le_os, - /// Unordered (non-signaling) - unord_q, - /// Not-equal (unordered, non-signaling) - neq_uq, - /// Not-less-than (unordered, signaling) - nlt_us, - /// Not-less-than-or-equal (unordered, signaling) - nle_us, - /// Ordered (non-signaling) - ord_q, - /// Equal (unordered, non-signaling) - eq_uq, - /// Not-greater-than-or-equal (unordered, signaling) - nge_us, - /// Not-greater-than (unordered, signaling) - ngt_us, - /// False (ordered, non-signaling) - false_oq, - /// Not-equal (ordered, non-signaling) - neq_oq, - /// Greater-than-or-equal (ordered, signaling) - ge_os, - /// Greater-than (ordered, signaling) - gt_os, - /// True (unordered, non-signaling) - true_uq, - /// Equal (unordered, non-signaling) - eq_os, - /// Less-than (ordered, non-signaling) - lt_oq, - /// Less-than-or-equal (ordered, non-signaling) - le_oq, - /// Unordered (signaling) - unord_s, - /// Not-equal (unordered, signaling) - neq_us, - /// Not-less-than (unordered, non-signaling) - nlt_uq, - /// Not-less-than-or-equal (unordered, non-signaling) - nle_uq, - /// Ordered (signaling) - ord_s, - /// Equal (unordered, signaling) - eq_us, - /// Not-greater-than-or-equal (unordered, non-signaling) - nge_uq, - /// Not-greater-than (unordered, non-signaling) - ngt_uq, - /// False (ordered, signaling) - false_os, - /// Not-equal (ordered, signaling) - neq_os, - /// Greater-than-or-equal (ordered, non-signaling) - ge_oq, - /// Greater-than (ordered, non-signaling) - gt_oq, - /// True (unordered, signaling) - true_us, - - /// Equal (ordered, non-signaling) - pub const eq: VexFloatPredicate = .eq_oq; - /// Less-than (ordered, signaling) - pub const lt: VexFloatPredicate = .lt_os; - /// Less-than-or-equal (ordered, signaling) - pub const le: VexFloatPredicate = .le_os; - /// Unordered (non-signaling) - pub const unord: VexFloatPredicate = .unord_q; - /// Not-equal (unordered, non-signaling) - pub const neq: VexFloatPredicate = .neq_uq; - /// Not-less-than (unordered, signaling) - pub const nlt: VexFloatPredicate = .nlt_us; - /// Not-less-than-or-equal (unordered, signaling) - pub const nle: VexFloatPredicate = .nle_us; - /// Ordered (non-signaling) - pub const ord: VexFloatPredicate = .ord_q; - /// Not-greater-than-or-equal (unordered, signaling) - pub const nge: VexFloatPredicate = .nge_us; - /// Not-greater-than (unordered, signaling) - pub const ngt: VexFloatPredicate = .ngt_us; - /// False (ordered, non-signaling) - pub const @"false": VexFloatPredicate = .false_oq; - /// Greater-than-or-equal (ordered, signaling) - pub const ge: VexFloatPredicate = .ge_os; - /// Greater-than (ordered, signaling) - pub const gt: VexFloatPredicate = .gt_os; - /// True (unordered, non-signaling) - pub const @"true": VexFloatPredicate = .true_uq; - - pub fn imm(pred: VexFloatPredicate) Immediate { - return .u(@intFromEnum(pred)); - } -}; - -pub const Register = enum(u8) { - // zig fmt: off - rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi, - r8, r9, r10, r11, r12, r13, r14, r15, - - eax, ecx, edx, ebx, esp, ebp, esi, edi, - r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d, - - ax, cx, dx, bx, sp, bp, si, di, - r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w, - - al, cl, dl, bl, spl, bpl, sil, dil, - r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b, - - ah, ch, dh, bh, - - zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, - zmm8, zmm9, zmm10, zmm11, zmm12, zmm13, zmm14, zmm15, - zmm16, zmm17,zmm18, zmm19, zmm20, zmm21, zmm22, zmm23, - zmm24, zmm25,zmm26, zmm27, zmm28, zmm29, zmm30, zmm31, - - ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, - ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15, - ymm16, ymm17,ymm18, ymm19, ymm20, ymm21, ymm22, ymm23, - ymm24, ymm25,ymm26, ymm27, ymm28, ymm29, ymm30, ymm31, - - xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, - xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, - xmm16, xmm17,xmm18, xmm19, xmm20, xmm21, xmm22, xmm23, - xmm24, xmm25,xmm26, xmm27, xmm28, xmm29, xmm30, xmm31, - - mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, - - st0, st1, st2, st3, st4, st5, st6, st7, - - es, cs, ss, ds, fs, gs, - - rip, eip, ip, - - cr0, cr1, cr2, cr3, cr4, cr5, cr6, cr7, - cr8, cr9, cr10, cr11, cr12, cr13, cr14, cr15, - - dr0, dr1, dr2, dr3, dr4, dr5, dr6, dr7, - dr8, dr9, dr10, dr11, dr12, dr13, dr14, dr15, - - none, - // zig fmt: on - - pub const Class = enum { - general_purpose, - gphi, - segment, - x87, - mmx, - sse, - ip, - cr, - dr, - }; - - pub fn class(reg: Register) Class { - return switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => .general_purpose, - @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => .general_purpose, - @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => .general_purpose, - @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => .general_purpose, - @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => .gphi, - - @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm31) => .sse, - @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm31) => .sse, - @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm31) => .sse, - @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => .mmx, - @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => .x87, - - @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => .segment, - @intFromEnum(Register.rip) ... @intFromEnum(Register.ip) => .ip, - @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => .cr, - @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => .dr, - - else => unreachable, - // zig fmt: on - }; - } - - pub inline fn isClass(reg: Register, rc: Class) bool { - switch (rc) { - else => return reg.class() == rc, - .gphi => { - const reg_id = reg.id(); - return (reg_id >= comptime Register.ah.id()) and reg_id <= comptime Register.bh.id(); - }, - } - } - - pub fn id(reg: Register) u7 { - const base = switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => @intFromEnum(Register.rax), - @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), - @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), - @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), - @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah), - - @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm31) => @intFromEnum(Register.zmm0) - 16, - @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm31) => @intFromEnum(Register.ymm0) - 16, - @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm31) => @intFromEnum(Register.xmm0) - 16, - @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => @intFromEnum(Register.mm0) - 48, - @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => @intFromEnum(Register.st0) - 56, - @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => @intFromEnum(Register.es) - 64, - @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => @intFromEnum(Register.cr0) - 70, - @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => @intFromEnum(Register.dr0) - 86, - - else => unreachable, - // zig fmt: on - }; - return @intCast(@intFromEnum(reg) - base); - } - - pub fn size(reg: Register) Memory.Size { - return switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => .qword, - @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => .dword, - @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => .word, - @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => .byte, - @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => .byte, - - @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm15) => .zword, - @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => .yword, - @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => .xword, - @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => .qword, - @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => .tbyte, - - @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => .word, - - @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => .gpr, - @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => .gpr, - - else => unreachable, - // zig fmt: on - }; - } - - pub fn isExtended(reg: Register) bool { - return switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.r8) ... @intFromEnum(Register.r15) => true, - @intFromEnum(Register.r8d) ... @intFromEnum(Register.r15d) => true, - @intFromEnum(Register.r8w) ... @intFromEnum(Register.r15w) => true, - @intFromEnum(Register.r8b) ... @intFromEnum(Register.r15b) => true, - - @intFromEnum(Register.zmm8) ... @intFromEnum(Register.zmm31) => true, - @intFromEnum(Register.ymm8) ... @intFromEnum(Register.ymm31) => true, - @intFromEnum(Register.xmm8) ... @intFromEnum(Register.xmm31) => true, - - @intFromEnum(Register.cr8) ... @intFromEnum(Register.cr15) => true, - @intFromEnum(Register.dr8) ... @intFromEnum(Register.dr15) => true, - - else => false, - // zig fmt: on - }; - } - - pub fn enc(reg: Register) u5 { - const base = switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => @intFromEnum(Register.rax), - @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), - @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), - @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), - @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah) - 4, - - @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0), - @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0), - @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => @intFromEnum(Register.mm0), - @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => @intFromEnum(Register.st0), - - @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => @intFromEnum(Register.es), - - @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => @intFromEnum(Register.cr0), - @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => @intFromEnum(Register.dr0), - - else => unreachable, - // zig fmt: on - }; - return @truncate(@intFromEnum(reg) - base); - } - - pub fn toBitSize(reg: Register, bit_size: u64) Register { - return switch (bit_size) { - 8 => reg.to8(), - 16 => reg.to16(), - 32 => reg.to32(), - 64 => reg.to64(), - 80 => reg.to80(), - 128 => reg.to128(), - 256 => reg.to256(), - 512 => reg.to512(), - else => unreachable, - }; - } - - pub fn toSize(reg: Register, new_size: Memory.Size, target: *const std.Target) Register { - return switch (new_size) { - .none => unreachable, - .ptr => reg.toBitSize(target.ptrBitWidth()), - .gpr => switch (target.cpu.arch) { - else => unreachable, - .x86 => reg.to32(), - .x86_64 => reg.to64(), - }, - .low_byte => reg.toLo8(), - .high_byte => reg.toHi8(), - .byte => reg.to8(), - .word => reg.to16(), - .dword => reg.to32(), - .qword => reg.to64(), - .tbyte => reg.to80(), - .xword => reg.to128(), - .yword => reg.to256(), - .zword => reg.to512(), - }; - } - - fn gpBase(reg: Register) u7 { - return switch (@intFromEnum(reg)) { - // zig fmt: off - @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => @intFromEnum(Register.rax), - @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), - @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), - @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), - @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah), - else => unreachable, - // zig fmt: on - }; - } - - pub fn to64(reg: Register) Register { - return switch (reg.class()) { - .general_purpose, .gphi => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.rax)), - .segment => unreachable, - .x87, .mmx, .cr, .dr => reg, - .sse => reg.to128(), - .ip => .rip, - }; - } - - pub fn to32(reg: Register) Register { - return switch (reg.class()) { - .general_purpose, .gphi => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.eax)), - .segment => unreachable, - .x87, .mmx, .cr, .dr => reg, - .sse => reg.to128(), - .ip => .eip, - }; - } - - pub fn to16(reg: Register) Register { - return switch (reg.class()) { - .general_purpose, .gphi => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.ax)), - .segment, .x87, .mmx, .cr, .dr => reg, - .sse => reg.to128(), - .ip => .ip, - }; - } - - pub fn to8(reg: Register) Register { - return switch (reg.class()) { - .general_purpose => reg.toLo8(), - .gphi, .segment, .x87, .mmx, .cr, .dr => reg, - .sse => reg.to128(), - .ip => .ip, - }; - } - - pub fn toLo8(reg: Register) Register { - return @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.al)); - } - - pub fn toHi8(reg: Register) Register { - assert(reg.isClass(.gphi)); - return @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.ah)); - } - - pub fn to80(reg: Register) Register { - assert(reg.isClass(.x87)); - return reg; - } - - fn sseBase(reg: Register) u8 { - assert(reg.isClass(.sse)); - return switch (@intFromEnum(reg)) { - @intFromEnum(Register.zmm0)...@intFromEnum(Register.zmm31) => @intFromEnum(Register.zmm0), - @intFromEnum(Register.ymm0)...@intFromEnum(Register.ymm31) => @intFromEnum(Register.ymm0), - @intFromEnum(Register.xmm0)...@intFromEnum(Register.xmm31) => @intFromEnum(Register.xmm0), - else => unreachable, - }; - } - - pub fn to512(reg: Register) Register { - return @enumFromInt(@intFromEnum(reg) - reg.sseBase() + @intFromEnum(Register.zmm0)); - } - - pub fn to256(reg: Register) Register { - return @enumFromInt(@intFromEnum(reg) - reg.sseBase() + @intFromEnum(Register.ymm0)); - } - - pub fn to128(reg: Register) Register { - return @enumFromInt(@intFromEnum(reg) - reg.sseBase() + @intFromEnum(Register.xmm0)); - } - - /// DWARF register encoding - pub fn dwarfNum(reg: Register) u6 { - return switch (reg.class()) { - .general_purpose, .gphi => if (reg.isExtended()) - reg.enc() - else - @as(u3, @truncate(@as(u24, 0o54673120) >> @as(u5, reg.enc()) * 3)), - .sse => 17 + @as(u6, reg.enc()), - .x87 => 33 + @as(u6, reg.enc()), - .mmx => 41 + @as(u6, reg.enc()), - .segment => 50 + @as(u6, reg.enc()), - .ip => 16, - .cr, .dr => unreachable, - }; - } -}; - -test "Register id - different classes" { - try expect(Register.al.id() == Register.ax.id()); - try expect(Register.ah.id() != Register.spl.id()); - try expect(Register.ax.id() == Register.eax.id()); - try expect(Register.eax.id() == Register.rax.id()); - - try expect(Register.ymm0.id() == 0b10000); - try expect(Register.ymm0.id() != Register.rax.id()); - try expect(Register.xmm0.id() == Register.ymm0.id()); - try expect(Register.xmm0.id() != Register.mm0.id()); - try expect(Register.mm0.id() != Register.st0.id()); -} - -test "Register enc - different classes" { - try expect(Register.al.enc() == Register.ax.enc()); - try expect(Register.ah.enc() == Register.spl.enc()); - try expect(Register.ax.enc() == Register.eax.enc()); - try expect(Register.eax.enc() == Register.rax.enc()); - try expect(Register.ymm0.enc() == Register.rax.enc()); - try expect(Register.xmm0.enc() == Register.ymm0.enc()); - try expect(Register.es.enc() == Register.rax.enc()); -} - -test "Register classes" { - try expect(Register.r11.isClass(.general_purpose)); - try expect(Register.rdx.isClass(.gphi)); - try expect(!Register.dil.isClass(.gphi)); - try expect(Register.ymm11.isClass(.sse)); - try expect(Register.mm3.isClass(.mmx)); - try expect(Register.st3.isClass(.x87)); - try expect(Register.fs.isClass(.segment)); -} - -pub const FrameIndex = enum(u32) { - // This index refers to the start of the arguments passed to this function - args_frame, - // This index refers to the return address pushed by a `call` and popped by a `ret`. - ret_addr, - // This index refers to the base pointer pushed in the prologue and popped in the epilogue. - base_ptr, - // This index refers to the entire stack frame. - stack_frame, - // This index refers to the start of the call frame for arguments passed to called functions - call_frame, - // Other indices are used for local variable stack slots - _, - - pub const named_count = @typeInfo(FrameIndex).@"enum".fields.len; - - pub fn isNamed(fi: FrameIndex) bool { - return @intFromEnum(fi) < named_count; - } - - pub fn format(fi: FrameIndex, writer: *std.Io.Writer) std.Io.Writer.Error!void { - if (fi.isNamed()) { - try writer.print("FrameIndex.{t}", .{fi}); - } else { - try writer.print("FrameIndex({d})", .{@intFromEnum(fi)}); - } - } -}; - -pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 }; - -pub const RegisterOffset = struct { reg: Register, off: i32 = 0 }; - -pub const NavOffset = struct { index: InternPool.Nav.Index, off: i32 = 0 }; - -pub const Memory = struct { - base: Base = .none, - mod: Mod = .{ .rm = .{} }, - - pub const Base = union(enum(u4)) { - none, - reg: Register, - frame: FrameIndex, - table, - rip_inst: Mir.Inst.Index, - nav: InternPool.Nav.Index, - uav: InternPool.Key.Ptr.BaseAddr.Uav, - lazy_sym: link.File.LazySymbol, - extern_func: Mir.NullTerminatedString, - - pub const Tag = @typeInfo(Base).@"union".tag_type.?; - }; - - pub const Mod = union(enum(u1)) { - rm: Rm, - off: u64, - - pub const Rm = struct { - size: Size = .none, - index: Register = .none, - scale: Scale = .@"1", - disp: i32 = 0, - }; - }; - - pub const Size = enum(u4) { - none, - ptr, - gpr, - low_byte, - high_byte, - byte, - word, - dword, - qword, - tbyte, - xword, - yword, - zword, - - pub fn fromSize(size: u32) Size { - return switch (size) { - 1...1 => .byte, - 2...2 => .word, - 3...4 => .dword, - 5...8 => .qword, - 9...16 => .xword, - 17...32 => .yword, - 33...64 => .zword, - else => unreachable, - }; - } - - pub fn fromBitSize(bit_size: u64) Size { - return switch (bit_size) { - 8 => .byte, - 16 => .word, - 32 => .dword, - 64 => .qword, - 80 => .tbyte, - 128 => .xword, - 256 => .yword, - 512 => .zword, - else => unreachable, - }; - } - - pub fn bitSize(s: Size, target: *const std.Target) u64 { - return switch (s) { - .none => 0, - .ptr => target.ptrBitWidth(), - .gpr => switch (target.cpu.arch) { - else => unreachable, - .x86 => 32, - .x86_64 => 64, - }, - .low_byte, .high_byte, .byte => 8, - .word => 16, - .dword => 32, - .qword => 64, - .tbyte => 80, - .xword => 128, - .yword => 256, - .zword => 512, - }; - } - - pub fn format(s: Size, writer: *std.Io.Writer) std.Io.Writer.Error!void { - if (s == .none) return; - try writer.writeAll(@tagName(s)); - switch (s) { - .none => unreachable, - .ptr, .gpr => {}, - else => { - try writer.writeByte(' '); - try writer.writeAll("ptr"); - }, - } - } - }; - - pub const Scale = enum(u2) { - @"1", - @"2", - @"4", - @"8", - - pub fn fromFactor(factor: u4) Scale { - return switch (factor) { - else => unreachable, - 1 => .@"1", - 2 => .@"2", - 4 => .@"4", - 8 => .@"8", - }; - } - - pub fn toFactor(scale: Scale) u4 { - return switch (scale) { - .@"1" => 1, - .@"2" => 2, - .@"4" => 4, - .@"8" => 8, - }; - } - - pub fn fromLog2(log2: u2) Scale { - return @enumFromInt(log2); - } - - pub fn toLog2(scale: Scale) u2 { - return @intFromEnum(scale); - } - }; -}; - -pub const Immediate = union(enum) { - signed: i32, - unsigned: u64, - nav: NavOffset, - uav: InternPool.Key.Ptr.BaseAddr.Uav, - lazy_sym: link.File.LazySymbol, - extern_func: Mir.NullTerminatedString, - - pub fn u(x: u64) Immediate { - return .{ .unsigned = x }; - } - - pub fn s(x: i32) Immediate { - return .{ .signed = x }; - } - - pub fn format(imm: Immediate, writer: *std.Io.Writer) std.Io.Writer.Error!void { - switch (imm) { - inline else => |int| try writer.print("{d}", .{int}), - .nav => |nav_off| try writer.print("Nav({d}) + {d}", .{ @intFromEnum(nav_off.nav), nav_off.off }), - .uav => |uav| try writer.print("Uav({d})", .{@intFromEnum(uav.val)}), - .lazy_sym => |lazy_sym| try writer.print("LazySym({s}, {d})", .{ @tagName(lazy_sym.kind), @intFromEnum(lazy_sym.ty) }), - .extern_func => |extern_func| try writer.print("ExternFunc({d})", .{@intFromEnum(extern_func)}), - } - } -}; diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig deleted file mode 100644 index 1d9c67175ef204933a5463393cad189d0e59e7fa..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/encoder.zig +++ /dev/null @@ -1,2789 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const log = std.log.scoped(.x86_64_encoder); -const math = std.math; -const testing = std.testing; -const Writer = std.Io.Writer; - -const bits = @import("bits.zig"); -const Encoding = @import("Encoding.zig"); -const FrameIndex = bits.FrameIndex; -const Register = bits.Register; -const Symbol = bits.Symbol; - -pub const Instruction = struct { - prefix: Prefix = .none, - encoding: Encoding, - ops: [4]Operand = .{.none} ** 4, - - pub const Mnemonic = Encoding.Mnemonic; - - pub const Prefix = enum(u3) { - none, - lock, - rep, - repe, - repz, - repne, - repnz, - directive, - }; - - pub const Immediate = union(enum) { - signed: i32, - unsigned: u64, - - pub fn u(x: u64) Immediate { - return .{ .unsigned = x }; - } - - pub fn s(x: i32) Immediate { - return .{ .signed = x }; - } - - pub fn asSigned(imm: Immediate, bit_size: u64) i64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(i8, @intCast(x)), - 16 => @as(i16, @intCast(x)), - 32, 64 => x, - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), - 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), - 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), - 64 => @bitCast(x), - else => unreachable, - }, - }; - } - - pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))), - 16 => @as(u16, @bitCast(@as(i16, @intCast(x)))), - 32, 64 => @as(u32, @bitCast(x)), - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(u8, @intCast(x)), - 16 => @as(u16, @intCast(x)), - 32 => @as(u32, @intCast(x)), - 64 => x, - else => unreachable, - }, - }; - } - }; - - pub const Memory = union(enum) { - sib: Sib, - rip: Rip, - moffs: Moffs, - - pub const Base = bits.Memory.Base; - - pub const ScaleIndex = struct { - scale: u4, - index: Register, - - const none = ScaleIndex{ .scale = 0, .index = undefined }; - }; - - pub const PtrSize = bits.Memory.Size; - - pub const Sib = struct { - ptr_size: PtrSize, - base: Base, - scale_index: ScaleIndex, - disp: i32, - }; - - pub const Rip = struct { - ptr_size: PtrSize, - disp: i32, - }; - - pub const Moffs = struct { - seg: Register, - offset: u64, - }; - - pub fn initMoffs(reg: Register, offset: u64) Memory { - assert(reg.isClass(.segment)); - return .{ .moffs = .{ .seg = reg, .offset = offset } }; - } - - pub fn initSib(ptr_size: PtrSize, args: struct { - disp: i32 = 0, - base: Base = .none, - scale_index: ?ScaleIndex = null, - }) Memory { - if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale)); - return .{ .sib = .{ - .base = args.base, - .disp = args.disp, - .ptr_size = ptr_size, - .scale_index = if (args.scale_index) |si| si else ScaleIndex.none, - } }; - } - - pub fn initRip(ptr_size: PtrSize, displacement: i32) Memory { - return .{ .rip = .{ .ptr_size = ptr_size, .disp = displacement } }; - } - - pub fn isSegmentRegister(mem: Memory) bool { - return switch (mem) { - .moffs => true, - .rip => false, - .sib => |s| switch (s.base) { - .none, .frame, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => false, - .reg => |reg| reg.isClass(.segment), - }, - }; - } - - pub fn base(mem: Memory) Base { - return switch (mem) { - .moffs => |m| .{ .reg = m.seg }, - .sib => |s| s.base, - .rip => .none, - }; - } - - pub fn scaleIndex(mem: Memory) ?ScaleIndex { - return switch (mem) { - .moffs, .rip => null, - .sib => |s| if (s.scale_index.scale > 0) s.scale_index else null, - }; - } - - pub fn disp(mem: Memory) Immediate { - return switch (mem) { - .sib => |s| .s(s.disp), - .rip => |r| .s(r.disp), - .moffs => |m| .u(m.offset), - }; - } - - pub fn bitSize(mem: Memory, target: *const std.Target) u64 { - return switch (mem) { - .rip => |r| r.ptr_size.bitSize(target), - .sib => |s| s.ptr_size.bitSize(target), - .moffs => target.ptrBitWidth(), - }; - } - }; - - pub const Operand = union(enum) { - none, - reg: Register, - mem: Memory, - imm: Immediate, - bytes: []const u8, - - /// Returns the bitsize of the operand. - pub fn bitSize(op: Operand) u64 { - return switch (op) { - .none => unreachable, - .reg => |reg| reg.bitSize(), - .mem => |mem| mem.bitSize(), - .imm => unreachable, - .bytes => unreachable, - }; - } - - /// Returns true if the operand is a segment register. - /// Asserts the operand is either register or memory. - pub fn isSegmentRegister(op: Operand) bool { - return switch (op) { - .none => unreachable, - .reg => |reg| reg.isClass(.segment), - .mem => |mem| mem.isSegmentRegister(), - .imm => unreachable, - .bytes => unreachable, - }; - } - - pub fn baseExtEnc(op: Operand) u2 { - return switch (op) { - .none, .imm => 0b00, - .reg => |reg| @truncate(reg.enc() >> 3), - .mem => |mem| switch (mem.base()) { - .none, .frame, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => 0b00, // rsp, rbp, and rip are not extended - .reg => |reg| @truncate(reg.enc() >> 3), - }, - .bytes => unreachable, - }; - } - - pub fn indexExtEnc(op: Operand) u2 { - return switch (op) { - .none, .reg, .imm => 0b00, - .mem => |mem| if (mem.scaleIndex()) |si| @truncate(si.index.enc() >> 3) else 0b00, - .bytes => unreachable, - }; - } - - const Format = struct { - op: Operand, - enc_op: Encoding.Op, - - fn default(f: Format, w: *Writer) Writer.Error!void { - const op = f.op; - const enc_op = f.enc_op; - switch (op) { - .none => {}, - .reg => |reg| try w.writeAll(@tagName(reg)), - .mem => |mem| switch (mem) { - .rip => |rip| { - try w.print("{f} [rip", .{rip.ptr_size}); - if (rip.disp != 0) try w.print(" {c} 0x{x}", .{ - @as(u8, if (rip.disp < 0) '-' else '+'), - @abs(rip.disp), - }); - try w.writeByte(']'); - }, - .sib => |sib| { - try w.print("{f} ", .{sib.ptr_size}); - - if (mem.isSegmentRegister()) { - return w.print("{s}:0x{x}", .{ @tagName(sib.base.reg), sib.disp }); - } - - try w.writeByte('['); - - var any = true; - switch (sib.base) { - .none => any = false, - .reg => |reg| try w.print("{s}", .{@tagName(reg)}), - .frame => |frame_index| try w.print("{f}", .{frame_index}), - .table => try w.print("Table", .{}), - .rip_inst => |inst_index| try w.print("RipInst({d})", .{inst_index}), - .nav => |nav| try w.print("Nav({d})", .{@intFromEnum(nav)}), - .uav => |uav| try w.print("Uav({d})", .{@intFromEnum(uav.val)}), - .lazy_sym => |lazy_sym| try w.print("LazySym({s}, {d})", .{ - @tagName(lazy_sym.kind), - @intFromEnum(lazy_sym.ty), - }), - .extern_func => |extern_func| try w.print("ExternFunc({d})", .{@intFromEnum(extern_func)}), - } - if (mem.scaleIndex()) |si| { - if (any) try w.writeAll(" + "); - try w.print("{s} * {d}", .{ @tagName(si.index), si.scale }); - any = true; - } - if (sib.disp != 0 or !any) { - if (any) - try w.print(" {c} ", .{@as(u8, if (sib.disp < 0) '-' else '+')}) - else if (sib.disp < 0) - try w.writeByte('-'); - try w.print("0x{x}", .{@abs(sib.disp)}); - any = true; - } - - try w.writeByte(']'); - }, - .moffs => |moffs| try w.print("{s}:0x{x}", .{ - @tagName(moffs.seg), - moffs.offset, - }), - }, - .imm => |imm| if (enc_op.isSigned()) { - const imms = imm.asSigned(enc_op.immBitSize()); - if (imms < 0) try w.writeByte('-'); - try w.print("0x{x}", .{@abs(imms)}); - } else try w.print("0x{x}", .{imm.asUnsigned(enc_op.immBitSize())}), - .bytes => unreachable, - } - } - }; - - pub fn fmt(op: Operand, enc_op: Encoding.Op) std.fmt.Alt(Format, Format.default) { - return .{ .data = .{ .op = op, .enc_op = enc_op } }; - } - }; - - pub fn new( - prefix: Prefix, - mnemonic: Mnemonic, - ops: []const Operand, - target: *const std.Target, - ) !Instruction { - const encoding: Encoding = switch (prefix) { - else => (try Encoding.findByMnemonic(prefix, mnemonic, ops, target)) orelse { - log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{ - @tagName(prefix), - @tagName(mnemonic), - @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none), - @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none), - @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none), - @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none), - }); - return error.InvalidInstruction; - }, - .directive => .{ - .mnemonic = mnemonic, - .data = .{ - .op_en = .z, - .ops = .{ - if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none, - if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none, - if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none, - if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none, - }, - .opc_len = 0, - .opc = undefined, - .modrm_ext = 0, - .mode = .none, - .feature = .none, - }, - }, - }; - log.debug("selected encoding: {f}", .{encoding}); - - var inst: Instruction = .{ - .prefix = prefix, - .encoding = encoding, - .ops = [1]Operand{.none} ** 4, - }; - @memcpy(inst.ops[0..ops.len], ops); - return inst; - } - - pub fn format(inst: Instruction, w: *Writer) Writer.Error!void { - switch (inst.prefix) { - .none, .directive => {}, - else => try w.print("{s} ", .{@tagName(inst.prefix)}), - } - try w.print("{s}", .{@tagName(inst.encoding.mnemonic)}); - for (inst.ops, inst.encoding.data.ops, 0..) |op, enc, i| { - if (op == .none) break; - if (i > 0) try w.writeByte(','); - try w.print(" {f}", .{op.fmt(enc)}); - } - } - - pub fn encode(inst: Instruction, w: *Writer, comptime opts: Options) !void { - assert(inst.prefix != .directive); - const encoder: Encoder(opts) = .{ .w = w }; - const enc = inst.encoding; - const data = enc.data; - - try inst.encodeWait(encoder); - if (data.mode.isVex()) { - try inst.encodeVexPrefix(encoder); - const opc = inst.encoding.opcode(); - try encoder.opcode_1byte(opc[opc.len - 1]); - } else { - try inst.encodeLegacyPrefixes(encoder); - try inst.encodeMandatoryPrefix(encoder); - try inst.encodeRexPrefix(encoder); - try inst.encodeOpcode(encoder); - } - - switch (data.op_en) { - .z, .o, .zo, .oz => {}, - .i, .d => try encodeImm(inst.ops[0].imm, data.ops[0], encoder), - .zi, .oi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder), - .ii => { - try encodeImm(inst.ops[0].imm, data.ops[0], encoder); - try encodeImm(inst.ops[1].imm, data.ops[1], encoder); - }, - .fd => try encoder.imm64(inst.ops[1].mem.moffs.offset), - .td => try encoder.imm64(inst.ops[0].mem.moffs.offset), - else => { - const mem_op: Operand = switch (data.op_en) { - .ia => .{ .reg = .eax }, - .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0], - .rm, .rmi, .rm0, .vm, .vmi, .rmv => inst.ops[1], - .rvm, .rvmr, .rvmi => inst.ops[2], - else => unreachable, - }; - switch (mem_op) { - .reg => |reg| { - const rm: u3 = switch (data.op_en) { - .ia, .m, .mi, .m1, .mc, .vm, .vmi => enc.modRmExt(), - .mr, .mri, .mrc => @truncate(inst.ops[1].reg.enc()), - .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => @truncate(inst.ops[0].reg.enc()), - .mvr => @truncate(inst.ops[2].reg.enc()), - else => unreachable, - }; - try encoder.modRm_direct(rm, @truncate(reg.enc())); - }, - .mem => |mem| { - const op = switch (data.op_en) { - .m, .mi, .m1, .mc, .vm, .vmi => .none, - .mr, .mri, .mrc => inst.ops[1], - .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0], - .mvr => inst.ops[2], - else => unreachable, - }; - try encodeMemory(enc, mem, op, encoder); - }, - else => unreachable, - } - - switch (data.op_en) { - .ia => try encodeImm(inst.ops[0].imm, data.ops[0], encoder), - .mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder), - .rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder), - .rvmr => try encoder.imm8(@as(u8, @as(u4, @intCast(inst.ops[3].reg.enc()))) << 4), - .rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder), - else => {}, - } - }, - } - } - - fn encodeOpcode(inst: Instruction, encoder: anytype) !void { - const opcode = inst.encoding.opcode(); - const first = @intFromBool(inst.encoding.mandatoryPrefix() != null); - const final = opcode.len - 1; - for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte); - switch (inst.encoding.data.op_en) { - .o, .oz, .oi => try encoder.opcode_withReg(opcode[final], @truncate(inst.ops[0].reg.enc())), - .zo => try encoder.opcode_withReg(opcode[final], @truncate(inst.ops[1].reg.enc())), - else => try encoder.opcode_1byte(opcode[final]), - } - } - - fn encodeWait(inst: Instruction, encoder: anytype) !void { - switch (inst.encoding.data.mode) { - .wait => try encoder.opcode_1byte(0x9b), - else => {}, - } - } - - fn encodeLegacyPrefixes(inst: Instruction, encoder: anytype) !void { - const enc = inst.encoding; - const data = enc.data; - const op_en = data.op_en; - - var legacy = LegacyPrefixes{}; - - switch (inst.prefix) { - .none => {}, - .lock => legacy.prefix_f0 = true, - .repne, .repnz => legacy.prefix_f2 = true, - .rep, .repe, .repz => legacy.prefix_f3 = true, - .directive => unreachable, - } - - switch (data.mode) { - .short, .rex_short => legacy.set16BitOverride(), - else => {}, - } - - const segment_override: ?Register = switch (op_en) { - .z, .i, .zi, .ii, .ia, .o, .zo, .oz, .oi, .d => null, - .fd => inst.ops[1].mem.base().reg, - .td => inst.ops[0].mem.base().reg, - .rm, .rmi, .rm0 => if (inst.ops[1].isSegmentRegister()) - switch (inst.ops[1]) { - .reg => |reg| reg, - .mem => |mem| mem.base().reg, - else => unreachable, - } - else - null, - .m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister()) - switch (inst.ops[0]) { - .reg => |reg| reg, - .mem => |mem| mem.base().reg, - else => unreachable, - } - else - null, - .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable, - }; - if (segment_override) |seg| { - legacy.setSegmentOverride(seg); - } - - try encoder.legacyPrefixes(legacy); - } - - fn encodeRexPrefix(inst: Instruction, encoder: anytype) !void { - const op_en = inst.encoding.data.op_en; - - var rex = Rex{}; - rex.present = inst.encoding.data.mode == .rex; - rex.w = inst.encoding.data.mode == .long; - - switch (op_en) { - .z, .i, .zi, .ii, .ia, .fd, .td, .d => {}, - .o, .oz, .oi => rex.b = inst.ops[0].reg.enc() & 0b01000 != 0, - .zo => rex.b = inst.ops[1].reg.enc() & 0b01000 != 0, - .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .rmv => { - const r_op = switch (op_en) { - .rm, .rmi, .rm0, .rmv => inst.ops[0], - .mr, .mri, .mrc => inst.ops[1], - else => .none, - }; - const r_op_base_ext_enc = r_op.baseExtEnc(); - rex.r = r_op_base_ext_enc & 0b01 != 0; - assert(r_op_base_ext_enc & 0b10 == 0); - - const b_x_op = switch (op_en) { - .rm, .rmi, .rm0 => inst.ops[1], - .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0], - else => unreachable, - }; - const b_x_op_base_ext_enc = b_x_op.baseExtEnc(); - rex.b = b_x_op_base_ext_enc & 0b01 != 0; - assert(b_x_op_base_ext_enc & 0b10 == 0); - const b_x_op_index_ext_enc = b_x_op.indexExtEnc(); - rex.x = b_x_op_index_ext_enc & 0b01 != 0; - assert(b_x_op_index_ext_enc & 0b10 == 0); - }, - .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable, - } - - try encoder.rex(rex); - } - - fn encodeVexPrefix(inst: Instruction, encoder: anytype) !void { - const op_en = inst.encoding.data.op_en; - const opc = inst.encoding.opcode(); - const mand_pre = inst.encoding.mandatoryPrefix(); - - var vex = Vex{}; - - vex.w = inst.encoding.data.mode.isLong(); - - switch (op_en) { - .z, .i, .zi, .ii, .ia, .fd, .td, .d, .o, .oz, .oi, .zo => unreachable, - .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => { - const r_op = switch (op_en) { - .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0], - .mr, .mri, .mrc => inst.ops[1], - .mvr => inst.ops[2], - .m, .mi, .m1, .mc, .vm, .vmi => .none, - else => unreachable, - }; - const r_op_base_ext_enc = r_op.baseExtEnc(); - vex.r = r_op_base_ext_enc & 0b01 != 0; - assert(r_op_base_ext_enc & 0b10 == 0); - - const b_x_op = switch (op_en) { - .rm, .rmi, .rm0, .vm, .vmi, .rmv => inst.ops[1], - .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0], - .rvm, .rvmr, .rvmi => inst.ops[2], - else => unreachable, - }; - const b_x_op_base_ext_enc = b_x_op.baseExtEnc(); - vex.b = b_x_op_base_ext_enc & 0b01 != 0; - assert(b_x_op_base_ext_enc & 0b10 == 0); - const b_x_op_index_ext_enc = b_x_op.indexExtEnc(); - vex.x = b_x_op_index_ext_enc & 0b01 != 0; - assert(b_x_op_index_ext_enc & 0b10 == 0); - }, - } - - vex.l = inst.encoding.data.mode.isVecLong(); - - vex.p = if (mand_pre) |mand| switch (mand) { - 0x66 => .@"66", - 0xf2 => .f2, - 0xf3 => .f3, - else => unreachable, - } else .none; - - const leading: usize = if (mand_pre) |_| 1 else 0; - assert(opc[leading] == 0x0f); - vex.m = switch (opc[leading + 1]) { - else => .@"0f", - 0x38 => .@"0f38", - 0x3a => .@"0f3a", - }; - - switch (op_en) { - else => {}, - .vm, .vmi => vex.v = inst.ops[0].reg, - .rvm, .rvmr, .rvmi => vex.v = inst.ops[1].reg, - .rmv => vex.v = inst.ops[2].reg, - } - - try encoder.vex(vex); - } - - fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void { - const prefix = inst.encoding.mandatoryPrefix() orelse return; - try encoder.opcode_1byte(prefix); - } - - fn encodeMemory(encoding: Encoding, mem: Memory, operand: Operand, encoder: anytype) !void { - const operand_enc: u3 = switch (operand) { - .reg => |reg| @truncate(reg.enc()), - .none => encoding.modRmExt(), - else => unreachable, - }; - - switch (mem) { - .moffs => unreachable, - .sib => |sib| switch (sib.base) { - .none, .table => { - try encoder.modRm_SIBDisp0(operand_enc); - if (mem.scaleIndex()) |si| { - const scale = math.log2_int(u4, si.scale); - try encoder.sib_scaleIndexDisp32(scale, @truncate(si.index.enc())); - } else { - try encoder.sib_disp32(); - } - try encoder.disp32(sib.disp); - }, - .reg => |base| switch (base.class()) { - .segment => { - // TODO audit this wrt SIB - try encoder.modRm_SIBDisp0(operand_enc); - if (mem.scaleIndex()) |si| { - const scale = math.log2_int(u4, si.scale); - try encoder.sib_scaleIndexDisp32(scale, @truncate(si.index.enc())); - } else { - try encoder.sib_disp32(); - } - try encoder.disp32(sib.disp); - }, - .general_purpose => { - const dst: u3 = @truncate(base.enc()); - const src = operand_enc; - if (dst == 4 or mem.scaleIndex() != null) { - if (sib.disp == 0 and dst != 5) { - try encoder.modRm_SIBDisp0(src); - if (mem.scaleIndex()) |si| { - const scale = math.log2_int(u4, si.scale); - try encoder.sib_scaleIndexBase(scale, @truncate(si.index.enc()), dst); - } else { - try encoder.sib_base(dst); - } - } else if (math.cast(i8, sib.disp)) |_| { - try encoder.modRm_SIBDisp8(src); - if (mem.scaleIndex()) |si| { - const scale = math.log2_int(u4, si.scale); - try encoder.sib_scaleIndexBaseDisp8(scale, @truncate(si.index.enc()), dst); - } else { - try encoder.sib_baseDisp8(dst); - } - try encoder.disp8(@as(i8, @truncate(sib.disp))); - } else { - try encoder.modRm_SIBDisp32(src); - if (mem.scaleIndex()) |si| { - const scale = math.log2_int(u4, si.scale); - try encoder.sib_scaleIndexBaseDisp32(scale, @truncate(si.index.enc()), dst); - } else { - try encoder.sib_baseDisp32(dst); - } - try encoder.disp32(sib.disp); - } - } else { - if (sib.disp == 0 and dst != 5) { - try encoder.modRm_indirectDisp0(src, dst); - } else if (math.cast(i8, sib.disp)) |_| { - try encoder.modRm_indirectDisp8(src, dst); - try encoder.disp8(@as(i8, @truncate(sib.disp))); - } else { - try encoder.modRm_indirectDisp32(src, dst); - try encoder.disp32(sib.disp); - } - } - }, - else => unreachable, - }, - .frame => if (@TypeOf(encoder).options.allow_frame_locs) { - try encoder.modRm_indirectDisp32(operand_enc, 0); - try encoder.disp32(undefined); - } else return error.CannotEncode, - .nav, .uav, .lazy_sym, .extern_func => if (@TypeOf(encoder).options.allow_symbols) { - try encoder.modRm_indirectDisp32(operand_enc, 0); - try encoder.disp32(undefined); - } else return error.CannotEncode, - .rip_inst => { - try encoder.modRm_RIPDisp32(operand_enc); - try encoder.disp32(sib.disp); - }, - }, - .rip => |rip| { - try encoder.modRm_RIPDisp32(operand_enc); - try encoder.disp32(rip.disp); - }, - } - } - - fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void { - const raw = imm.asUnsigned(kind.immBitSize()); - switch (kind.immBitSize()) { - 8 => try encoder.imm8(@as(u8, @intCast(raw))), - 16 => try encoder.imm16(@as(u16, @intCast(raw))), - 32 => try encoder.imm32(@as(u32, @intCast(raw))), - 64 => try encoder.imm64(raw), - else => unreachable, - } - } -}; - -pub const LegacyPrefixes = packed struct { - /// LOCK - prefix_f0: bool = false, - /// REPNZ, REPNE, REP, Scalar Double-precision - prefix_f2: bool = false, - /// REPZ, REPE, REP, Scalar Single-precision - prefix_f3: bool = false, - - /// CS segment override or Branch not taken - prefix_2e: bool = false, - /// SS segment override - prefix_36: bool = false, - /// ES segment override - prefix_26: bool = false, - /// FS segment override - prefix_64: bool = false, - /// GS segment override - prefix_65: bool = false, - - /// Branch taken - prefix_3e: bool = false, - - /// Address size override (enables 16 bit address size) - prefix_67: bool = false, - - /// Operand size override (enables 16 bit operation) - prefix_66: bool = false, - - padding: u5 = 0, - - pub fn setSegmentOverride(self: *LegacyPrefixes, reg: Register) void { - assert(reg.isClass(.segment)); - switch (reg) { - .cs => self.prefix_2e = true, - .ss => self.prefix_36 = true, - .es => self.prefix_26 = true, - .fs => self.prefix_64 = true, - .gs => self.prefix_65 = true, - .ds => {}, - else => unreachable, - } - } - - pub fn set16BitOverride(self: *LegacyPrefixes) void { - self.prefix_66 = true; - } -}; - -pub const Options = struct { allow_frame_locs: bool = false, allow_symbols: bool = false }; - -fn Encoder(comptime opts: Options) type { - return struct { - w: *Writer, - - const Self = @This(); - pub const options = opts; - - // -------- - // Prefixes - // -------- - - /// Encodes legacy prefixes - pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) !void { - if (@as(u16, @bitCast(prefixes)) != 0) { - // Hopefully this path isn't taken very often, so we'll do it the slow way for now - - // LOCK - if (prefixes.prefix_f0) try self.w.writeByte(0xf0); - // REPNZ, REPNE, REP, Scalar Double-precision - if (prefixes.prefix_f2) try self.w.writeByte(0xf2); - // REPZ, REPE, REP, Scalar Single-precision - if (prefixes.prefix_f3) try self.w.writeByte(0xf3); - - // CS segment override or Branch not taken - if (prefixes.prefix_2e) try self.w.writeByte(0x2e); - // DS segment override - if (prefixes.prefix_36) try self.w.writeByte(0x36); - // ES segment override - if (prefixes.prefix_26) try self.w.writeByte(0x26); - // FS segment override - if (prefixes.prefix_64) try self.w.writeByte(0x64); - // GS segment override - if (prefixes.prefix_65) try self.w.writeByte(0x65); - - // Branch taken - if (prefixes.prefix_3e) try self.w.writeByte(0x3e); - - // Operand size override - if (prefixes.prefix_66) try self.w.writeByte(0x66); - - // Address size override - if (prefixes.prefix_67) try self.w.writeByte(0x67); - } - } - - /// Use 16 bit operand size - /// - /// Note that this flag is overridden by REX.W, if both are present. - pub fn prefix16BitMode(self: Self) !void { - try self.w.writeByte(0x66); - } - - /// Encodes a REX prefix byte given all the fields - /// - /// Use this byte whenever you need 64 bit operation, - /// or one of reg, index, r/m, base, or opcode-reg might be extended. - /// - /// See struct `Rex` for a description of each field. - pub fn rex(self: Self, fields: Rex) !void { - if (!fields.present and !fields.isSet()) return; - - var byte: u8 = 0b0100_0000; - - if (fields.w) byte |= 0b1000; - if (fields.r) byte |= 0b0100; - if (fields.x) byte |= 0b0010; - if (fields.b) byte |= 0b0001; - - try self.w.writeByte(byte); - } - - /// Encodes a VEX prefix given all the fields - /// - /// See struct `Vex` for a description of each field. - pub fn vex(self: Self, fields: Vex) !void { - if (fields.is3Byte()) { - try self.w.writeByte(0b1100_0100); - - try self.w.writeByte( - @as(u8, ~@intFromBool(fields.r)) << 7 | - @as(u8, ~@intFromBool(fields.x)) << 6 | - @as(u8, ~@intFromBool(fields.b)) << 5 | - @as(u8, @intFromEnum(fields.m)) << 0, - ); - - try self.w.writeByte( - @as(u8, @intFromBool(fields.w)) << 7 | - @as(u8, ~@as(u4, @intCast(fields.v.enc()))) << 3 | - @as(u8, @intFromBool(fields.l)) << 2 | - @as(u8, @intFromEnum(fields.p)) << 0, - ); - } else { - try self.w.writeByte(0b1100_0101); - try self.w.writeByte( - @as(u8, ~@intFromBool(fields.r)) << 7 | - @as(u8, ~@as(u4, @intCast(fields.v.enc()))) << 3 | - @as(u8, @intFromBool(fields.l)) << 2 | - @as(u8, @intFromEnum(fields.p)) << 0, - ); - } - } - - // ------ - // Opcode - // ------ - - /// Encodes a 1 byte opcode - pub fn opcode_1byte(self: Self, opcode: u8) !void { - try self.w.writeByte(opcode); - } - - /// Encodes a 2 byte opcode - /// - /// e.g. IMUL has the opcode 0x0f 0xaf, so you use - /// - /// encoder.opcode_2byte(0x0f, 0xaf); - pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) !void { - try self.w.writeAll(&.{ prefix, opcode }); - } - - /// Encodes a 3 byte opcode - /// - /// e.g. MOVSD has the opcode 0xf2 0x0f 0x10 - /// - /// encoder.opcode_3byte(0xf2, 0x0f, 0x10); - pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) !void { - try self.w.writeAll(&.{ prefix_1, prefix_2, opcode }); - } - - /// Encodes a 1 byte opcode with a reg field - /// - /// Remember to add a REX prefix byte if reg is extended! - pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) !void { - assert(opcode & 0b111 == 0); - try self.w.writeByte(opcode | reg); - } - - // ------ - // ModR/M - // ------ - - /// Construct a ModR/M byte given all the fields - /// - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) !void { - try self.w.writeByte(@as(u8, mod) << 6 | @as(u8, reg_or_opx) << 3 | rm); - } - - /// Construct a ModR/M byte using direct r/m addressing - /// r/m effective address: r/m - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) !void { - try self.modRm(0b11, reg_or_opx, rm); - } - - /// Construct a ModR/M byte using indirect r/m addressing - /// r/m effective address: [r/m] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) !void { - assert(rm != 4 and rm != 5); - try self.modRm(0b00, reg_or_opx, rm); - } - - /// Construct a ModR/M byte using indirect SIB addressing - /// r/m effective address: [SIB] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) !void { - try self.modRm(0b00, reg_or_opx, 0b100); - } - - /// Construct a ModR/M byte using RIP-relative addressing - /// r/m effective address: [RIP + disp32] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) !void { - try self.modRm(0b00, reg_or_opx, 0b101); - } - - /// Construct a ModR/M byte using indirect r/m with a 8bit displacement - /// r/m effective address: [r/m + disp8] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) !void { - assert(rm != 4); - try self.modRm(0b01, reg_or_opx, rm); - } - - /// Construct a ModR/M byte using indirect SIB with a 8bit displacement - /// r/m effective address: [SIB + disp8] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) !void { - try self.modRm(0b01, reg_or_opx, 0b100); - } - - /// Construct a ModR/M byte using indirect r/m with a 32bit displacement - /// r/m effective address: [r/m + disp32] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) !void { - assert(rm != 4); - try self.modRm(0b10, reg_or_opx, rm); - } - - /// Construct a ModR/M byte using indirect SIB with a 32bit displacement - /// r/m effective address: [SIB + disp32] - /// - /// Note reg's effective address is always just reg for the ModR/M byte. - /// Remember to add a REX prefix byte if reg or rm are extended! - pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) !void { - try self.modRm(0b10, reg_or_opx, 0b100); - } - - // --- - // SIB - // --- - - /// Construct a SIB byte given all the fields - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib(self: Self, scale: u2, index: u3, base: u3) !void { - try self.w.writeByte(@as(u8, scale) << 6 | @as(u8, index) << 3 | base); - } - - /// Construct a SIB byte with scale * index + base, no frills. - /// r/m effective address: [base + scale * index] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) !void { - assert(base != 5); - - try self.sib(scale, index, base); - } - - /// Construct a SIB byte with scale * index + disp32 - /// r/m effective address: [scale * index + disp32] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) !void { - // scale is actually ignored - // index = 4 means no index if and only if we haven't extended the register - // TODO enforce this - // base = 5 means no base, if mod == 0. - try self.sib(scale, index, 5); - } - - /// Construct a SIB byte with just base - /// r/m effective address: [base] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_base(self: Self, base: u3) !void { - assert(base != 5); - - // scale is actually ignored - // index = 4 means no index - try self.sib(0, 4, base); - } - - /// Construct a SIB byte with just disp32 - /// r/m effective address: [disp32] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_disp32(self: Self) !void { - // scale is actually ignored - // index = 4 means no index - // base = 5 means no base, if mod == 0. - try self.sib(0, 4, 5); - } - - /// Construct a SIB byte with scale * index + base + disp8 - /// r/m effective address: [base + scale * index + disp8] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) !void { - try self.sib(scale, index, base); - } - - /// Construct a SIB byte with base + disp8, no index - /// r/m effective address: [base + disp8] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_baseDisp8(self: Self, base: u3) !void { - // scale is ignored - // index = 4 means no index - try self.sib(0, 4, base); - } - - /// Construct a SIB byte with scale * index + base + disp32 - /// r/m effective address: [base + scale * index + disp32] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) !void { - try self.sib(scale, index, base); - } - - /// Construct a SIB byte with base + disp32, no index - /// r/m effective address: [base + disp32] - /// - /// Remember to add a REX prefix byte if index or base are extended! - pub fn sib_baseDisp32(self: Self, base: u3) !void { - // scale is ignored - // index = 4 means no index - try self.sib(0, 4, base); - } - - // ------------------------- - // Trivial (no bit fiddling) - // ------------------------- - - /// Encode an 8 bit displacement - /// - /// It is sign-extended to 64 bits by the cpu. - pub fn disp8(self: Self, disp: i8) !void { - try self.w.writeByte(@as(u8, @bitCast(disp))); - } - - /// Encode an 32 bit displacement - /// - /// It is sign-extended to 64 bits by the cpu. - pub fn disp32(self: Self, disp: i32) !void { - try self.w.writeInt(i32, disp, .little); - } - - /// Encode an 8 bit immediate - /// - /// It is sign-extended to 64 bits by the cpu. - pub fn imm8(self: Self, imm: u8) !void { - try self.w.writeByte(imm); - } - - /// Encode an 16 bit immediate - /// - /// It is sign-extended to 64 bits by the cpu. - pub fn imm16(self: Self, imm: u16) !void { - try self.w.writeInt(u16, imm, .little); - } - - /// Encode an 32 bit immediate - /// - /// It is sign-extended to 64 bits by the cpu. - pub fn imm32(self: Self, imm: u32) !void { - try self.w.writeInt(u32, imm, .little); - } - - /// Encode an 64 bit immediate - /// - /// It is sign-extended to 64 bits by the cpu. - pub fn imm64(self: Self, imm: u64) !void { - try self.w.writeInt(u64, imm, .little); - } - }; -} - -pub const Rex = struct { - w: bool = false, - r: bool = false, - x: bool = false, - b: bool = false, - present: bool = false, - - pub fn isSet(rex: Rex) bool { - return rex.w or rex.r or rex.x or rex.b; - } -}; - -pub const Vex = struct { - w: bool = false, - r: bool = false, - x: bool = false, - b: bool = false, - l: bool = false, - p: enum(u2) { - none = 0b00, - @"66" = 0b01, - f3 = 0b10, - f2 = 0b11, - } = .none, - m: enum(u5) { - @"0f" = 0b0_0001, - @"0f38" = 0b0_0010, - @"0f3a" = 0b0_0011, - _, - } = .@"0f", - v: Register = .ymm0, - - pub fn is3Byte(vex: Vex) bool { - return vex.w or vex.x or vex.b or vex.m != .@"0f"; - } -}; - -// Tests -fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void { - assert(expected.len > 0); - if (std.mem.eql(u8, expected, given)) return; - const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{expected}); - defer testing.allocator.free(expected_fmt); - const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{given}); - defer testing.allocator.free(given_fmt); - const idx = std.mem.indexOfDiff(u8, expected_fmt, given_fmt).?; - const padding = try testing.allocator.alloc(u8, idx + 5); - defer testing.allocator.free(padding); - @memset(padding, ' '); - std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{ - assembly, - expected_fmt, - given_fmt, - padding, - }); - return error.TestFailed; -} - -const TestEncode = struct { - buffer: [32]u8 = undefined, - index: usize = 0, - - fn encode( - enc: *TestEncode, - mnemonic: Instruction.Mnemonic, - ops: []const Instruction.Operand, - ) !void { - var writer: std.Io.Writer = .fixed(&enc.buffer); - const inst: Instruction = try .new(.none, mnemonic, ops); - try inst.encode(&writer, .{}); - enc.index = writer.bufferedLen(); - } - - fn code(enc: TestEncode) []const u8 { - return enc.buffer[0..enc.index]; - } -}; - -test "encode" { - var buf = std.array_list.Managed(u8).init(testing.allocator); - defer buf.deinit(); - - const inst: Instruction = try .new(.none, .mov, &.{ - .{ .reg = .rbx }, - .{ .imm = .u(4) }, - }); - try inst.encode(buf.writer(), .{}); - try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items); -} - -test "lower I encoding" { - var enc = TestEncode{}; - - try enc.encode(.push, &.{ - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10"); - - try enc.encode(.push, &.{ - .{ .imm = .u(0x1000) }, - }); - try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000"); - - try enc.encode(.push, &.{ - .{ .imm = .u(0x10000000) }, - }); - try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000"); - - try enc.encode(.adc, &.{ - .{ .reg = .rax }, - .{ .imm = .u(0x10000000) }, - }); - try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000"); - - try enc.encode(.add, &.{ - .{ .reg = .al }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10"); - - try enc.encode(.add, &.{ - .{ .reg = .rax }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); - - try enc.encode(.sbb, &.{ - .{ .reg = .ax }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10"); - - try enc.encode(.xor, &.{ - .{ .reg = .al }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10"); -} - -test "lower MI encoding" { - var enc = TestEncode{}; - - try enc.encode(.mov, &.{ - .{ .reg = .r12 }, - .{ .imm = .u(0x1000) }, - }); - try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .r12 } }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10"); - - try enc.encode(.mov, &.{ - .{ .reg = .r12 }, - .{ .imm = .u(0x1000) }, - }); - try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); - - try enc.encode(.mov, &.{ - .{ .reg = .r12 }, - .{ .imm = .u(0x1000) }, - }); - try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 } }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initRip(.qword, 0x10) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings( - "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00", - enc.code(), - "mov QWORD PTR [rip + 0x10], 0x10", - ); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -8 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -2 }) }, - .{ .imm = .s(-16) }, - }); - try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -1 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .{ .reg = .ds }, - .disp = 0x10000000, - .scale_index = .{ .scale = 2, .index = .rcx }, - }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings( - "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00", - enc.code(), - "mov QWORD PTR [rcx*2 + 0x10000000], 0x10", - ); - - try enc.encode(.adc, &.{ - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10"); - - try enc.encode(.adc, &.{ - .{ .mem = Instruction.Memory.initRip(.qword, 0) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10"); - - try enc.encode(.adc, &.{ - .{ .reg = .rax }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10"); - - try enc.encode(.add, &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .rdx }, .disp = -8 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10"); - - try enc.encode(.add, &.{ - .{ .reg = .rax }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); - - try enc.encode(.add, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, - .{ .imm = .s(-0x10) }, - }); - try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10"); - - try enc.encode(.@"and", &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings( - "\x83\x24\x25\x00\x00\x00\x10\x10", - enc.code(), - "and DWORD PTR ds:0x10000000, 0x10", - ); - - try enc.encode(.@"and", &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .es }, .disp = 0x10000000 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings( - "\x26\x83\x24\x25\x00\x00\x00\x10\x10", - enc.code(), - "and DWORD PTR es:0x10000000, 0x10", - ); - - try enc.encode(.@"and", &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings( - "\x41\x83\xA4\x24\x00\x00\x00\x10\x10", - enc.code(), - "and DWORD PTR [r12 + 0x10000000], 0x10", - ); - - try enc.encode(.sub, &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings( - "\x41\x83\xAB\x00\x00\x00\x10\x10", - enc.code(), - "sub DWORD PTR [r11 + 0x10000000], 0x10", - ); -} - -test "lower RM encoding" { - var enc = TestEncode{}; - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r11 } }) }, - }); - try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]"); - - try enc.encode(.mov, &.{ - .{ .reg = .rbx }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0x10 }) }, - }); - try expectEqualHexStrings("\x48\x8B\x1C\x25\x10\x00\x00\x00", enc.code(), "mov rbx, QWORD PTR ds:0x10"); - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -4 }) }, - }); - try expectEqualHexStrings("\x48\x8B\x45\xFC", enc.code(), "mov rax, QWORD PTR [rbp - 4]"); - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .{ .reg = .rbp }, - .scale_index = .{ .scale = 1, .index = .rcx }, - .disp = -8, - }) }, - }); - try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*1 - 8]"); - - try enc.encode(.mov, &.{ - .{ .reg = .eax }, - .{ .mem = Instruction.Memory.initSib(.dword, .{ - .base = .{ .reg = .rbp }, - .scale_index = .{ .scale = 4, .index = .rdx }, - .disp = -4, - }) }, - }); - try expectEqualHexStrings("\x8B\x44\x95\xFC", enc.code(), "mov eax, dword ptr [rbp + rdx*4 - 4]"); - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .{ .reg = .rbp }, - .scale_index = .{ .scale = 8, .index = .rcx }, - .disp = -8, - }) }, - }); - try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*8 - 8]"); - - try enc.encode(.mov, &.{ - .{ .reg = .r8b }, - .{ .mem = Instruction.Memory.initSib(.byte, .{ - .base = .{ .reg = .rsi }, - .scale_index = .{ .scale = 1, .index = .rcx }, - .disp = -24, - }) }, - }); - try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", enc.code(), "mov r8b, BYTE PTR [rsi + rcx*1 - 24]"); - - // TODO this mnemonic needs cleanup as some prefixes are obsolete. - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .reg = .cs }, - }); - try expectEqualHexStrings("\x48\x8C\xC8", enc.code(), "mov rax, cs"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .reg = .fs }, - }); - try expectEqualHexStrings("\x8C\x65\xF0", enc.code(), "mov WORD PTR [rbp - 16], fs"); - - try enc.encode(.mov, &.{ - .{ .reg = .r12w }, - .{ .reg = .cs }, - }); - try expectEqualHexStrings("\x66\x41\x8C\xCC", enc.code(), "mov r12w, cs"); - - try enc.encode(.movsx, &.{ - .{ .reg = .eax }, - .{ .reg = .bx }, - }); - try expectEqualHexStrings("\x0F\xBF\xC3", enc.code(), "movsx eax, bx"); - - try enc.encode(.movsx, &.{ - .{ .reg = .eax }, - .{ .reg = .bl }, - }); - try expectEqualHexStrings("\x0F\xBE\xC3", enc.code(), "movsx eax, bl"); - - try enc.encode(.movsx, &.{ - .{ .reg = .ax }, - .{ .reg = .bl }, - }); - try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl"); - - try enc.encode(.movsx, &.{ - .{ .reg = .eax }, - .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp } }) }, - }); - try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]"); - - try enc.encode(.movsx, &.{ - .{ .reg = .eax }, - .{ .mem = Instruction.Memory.initSib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) }, - }); - try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]"); - - try enc.encode(.movsx, &.{ - .{ .reg = .ax }, - .{ .mem = Instruction.Memory.initRip(.byte, 0x10) }, - }); - try expectEqualHexStrings("\x66\x0F\xBE\x05\x10\x00\x00\x00", enc.code(), "movsx ax, BYTE PTR [rip + 0x10]"); - - try enc.encode(.movsx, &.{ - .{ .reg = .rax }, - .{ .reg = .bx }, - }); - try expectEqualHexStrings("\x48\x0F\xBF\xC3", enc.code(), "movsx rax, bx"); - - try enc.encode(.movsxd, &.{ - .{ .reg = .rax }, - .{ .reg = .ebx }, - }); - try expectEqualHexStrings("\x48\x63\xC3", enc.code(), "movsxd rax, ebx"); - - try enc.encode(.lea, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initRip(.qword, 0x10) }, - }); - try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, QWORD PTR [rip + 0x10]"); - - try enc.encode(.lea, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initRip(.dword, 0x10) }, - }); - try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, DWORD PTR [rip + 0x10]"); - - try enc.encode(.lea, &.{ - .{ .reg = .eax }, - .{ .mem = Instruction.Memory.initRip(.dword, 0x10) }, - }); - try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, DWORD PTR [rip + 0x10]"); - - try enc.encode(.lea, &.{ - .{ .reg = .eax }, - .{ .mem = Instruction.Memory.initRip(.word, 0x10) }, - }); - try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, WORD PTR [rip + 0x10]"); - - try enc.encode(.lea, &.{ - .{ .reg = .ax }, - .{ .mem = Instruction.Memory.initRip(.byte, 0x10) }, - }); - try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]"); - - try enc.encode(.lea, &.{ - .{ .reg = .rsi }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .{ .reg = .rbp }, - .scale_index = .{ .scale = 1, .index = .rcx }, - }) }, - }); - try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]"); - - try enc.encode(.add, &.{ - .{ .reg = .r11 }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - }); - try expectEqualHexStrings("\x4C\x03\x1C\x25\x00\x00\x00\x10", enc.code(), "add r11, QWORD PTR ds:0x10000000"); - - try enc.encode(.add, &.{ - .{ .reg = .r12b }, - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - }); - try expectEqualHexStrings("\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR ds:0x10000000"); - - try enc.encode(.add, &.{ - .{ .reg = .r12b }, - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .fs }, .disp = 0x10000000 }) }, - }); - try expectEqualHexStrings("\x64\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR fs:0x10000000"); - - try enc.encode(.sub, &.{ - .{ .reg = .r11 }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r13 }, .disp = 0x10000000 }) }, - }); - try expectEqualHexStrings("\x4D\x2B\x9D\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r13 + 0x10000000]"); - - try enc.encode(.sub, &.{ - .{ .reg = .r11 }, - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) }, - }); - try expectEqualHexStrings("\x4D\x2B\x9C\x24\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r12 + 0x10000000]"); - - try enc.encode(.imul, &.{ - .{ .reg = .r11 }, - .{ .reg = .r12 }, - }); - try expectEqualHexStrings("\x4D\x0F\xAF\xDC", enc.code(), "mov r11, r12"); -} - -test "lower RMI encoding" { - var enc = TestEncode{}; - - try enc.encode(.imul, &.{ - .{ .reg = .r11 }, - .{ .reg = .r12 }, - .{ .imm = .s(-2) }, - }); - try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2"); - - try enc.encode(.imul, &.{ - .{ .reg = .r11 }, - .{ .mem = Instruction.Memory.initRip(.qword, -16) }, - .{ .imm = .s(-1024) }, - }); - try expectEqualHexStrings( - "\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF", - enc.code(), - "imul r11, QWORD PTR [rip - 16], -1024", - ); - - try enc.encode(.imul, &.{ - .{ .reg = .bx }, - .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .imm = .s(-1024) }, - }); - try expectEqualHexStrings( - "\x66\x69\x5D\xF0\x00\xFC", - enc.code(), - "imul bx, WORD PTR [rbp - 16], -1024", - ); - - try enc.encode(.imul, &.{ - .{ .reg = .bx }, - .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .imm = .u(1024) }, - }); - try expectEqualHexStrings( - "\x66\x69\x5D\xF0\x00\x04", - enc.code(), - "imul bx, WORD PTR [rbp - 16], 1024", - ); -} - -test "lower MR encoding" { - var enc = TestEncode{}; - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .reg = .rbx }, - }); - try expectEqualHexStrings("\x48\x89\xD8", enc.code(), "mov rax, rbx"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -4 }) }, - .{ .reg = .r11 }, - }); - try expectEqualHexStrings("\x4c\x89\x5d\xfc", enc.code(), "mov QWORD PTR [rbp - 4], r11"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initRip(.qword, 0x10) }, - .{ .reg = .r12 }, - }); - try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rip + 0x10], r12"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .{ .reg = .r11 }, - .scale_index = .{ .scale = 2, .index = .r12 }, - .disp = 0x10, - }) }, - .{ .reg = .r13 }, - }); - try expectEqualHexStrings("\x4F\x89\x6C\x63\x10", enc.code(), "mov QWORD PTR [r11 + 2 * r12 + 0x10], r13"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initRip(.word, -0x10) }, - .{ .reg = .r12w }, - }); - try expectEqualHexStrings("\x66\x44\x89\x25\xF0\xFF\xFF\xFF", enc.code(), "mov WORD PTR [rip - 0x10], r12w"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initSib(.byte, .{ - .base = .{ .reg = .r11 }, - .scale_index = .{ .scale = 2, .index = .r12 }, - .disp = 0x10, - }) }, - .{ .reg = .r13b }, - }); - try expectEqualHexStrings("\x47\x88\x6C\x63\x10", enc.code(), "mov BYTE PTR [r11 + 2 * r12 + 0x10], r13b"); - - try enc.encode(.add, &.{ - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - .{ .reg = .r12b }, - }); - try expectEqualHexStrings("\x44\x00\x24\x25\x00\x00\x00\x10", enc.code(), "add BYTE PTR ds:0x10000000, r12b"); - - try enc.encode(.add, &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - .{ .reg = .r12d }, - }); - try expectEqualHexStrings("\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [ds:0x10000000], r12d"); - - try enc.encode(.add, &.{ - .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .gs }, .disp = 0x10000000 }) }, - .{ .reg = .r12d }, - }); - try expectEqualHexStrings("\x65\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [gs:0x10000000], r12d"); - - try enc.encode(.sub, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) }, - .{ .reg = .r12 }, - }); - try expectEqualHexStrings("\x4D\x29\xA3\x00\x00\x00\x10", enc.code(), "sub QWORD PTR [r11 + 0x10000000], r12"); -} - -test "lower M encoding" { - var enc = TestEncode{}; - - try enc.encode(.call, &.{ - .{ .reg = .r12 }, - }); - try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12"); - - try enc.encode(.call, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r12 } }) }, - }); - try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]"); - - try enc.encode(.call, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .none, - .scale_index = .{ .index = .r11, .scale = 2 }, - }) }, - }); - try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]"); - - try enc.encode(.call, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ - .base = .none, - .scale_index = .{ .index = .r12, .scale = 2 }, - }) }, - }); - try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]"); - - try enc.encode(.call, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .gs } }) }, - }); - try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0"); - - try enc.encode(.call, &.{ - .{ .imm = .s(0) }, - }); - try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0"); - - try enc.encode(.push, &.{ - .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp } }) }, - }); - try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]"); - - try enc.encode(.push, &.{ - .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp } }) }, - }); - try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]"); - - try enc.encode(.pop, &.{ - .{ .mem = Instruction.Memory.initRip(.qword, 0) }, - }); - try expectEqualHexStrings("\x8F\x05\x00\x00\x00\x00", enc.code(), "pop QWORD PTR [rip]"); - - try enc.encode(.pop, &.{ - .{ .mem = Instruction.Memory.initRip(.word, 0) }, - }); - try expectEqualHexStrings("\x66\x8F\x05\x00\x00\x00\x00", enc.code(), "pop WORD PTR [rbp]"); - - try enc.encode(.imul, &.{ - .{ .reg = .rax }, - }); - try expectEqualHexStrings("\x48\xF7\xE8", enc.code(), "imul rax"); - - try enc.encode(.imul, &.{ - .{ .reg = .r12 }, - }); - try expectEqualHexStrings("\x49\xF7\xEC", enc.code(), "imul r12"); -} - -test "lower O encoding" { - var enc = TestEncode{}; - - try enc.encode(.push, &.{ - .{ .reg = .rax }, - }); - try expectEqualHexStrings("\x50", enc.code(), "push rax"); - - try enc.encode(.push, &.{ - .{ .reg = .r12w }, - }); - try expectEqualHexStrings("\x66\x41\x54", enc.code(), "push r12w"); - - try enc.encode(.pop, &.{ - .{ .reg = .r12 }, - }); - try expectEqualHexStrings("\x41\x5c", enc.code(), "pop r12"); -} - -test "lower OI encoding" { - var enc = TestEncode{}; - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .imm = .u(0x1000000000000000) }, - }); - try expectEqualHexStrings( - "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", - enc.code(), - "movabs rax, 0x1000000000000000", - ); - - try enc.encode(.mov, &.{ - .{ .reg = .r11 }, - .{ .imm = .u(0x1000000000000000) }, - }); - try expectEqualHexStrings( - "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", - enc.code(), - "movabs r11, 0x1000000000000000", - ); - - try enc.encode(.mov, &.{ - .{ .reg = .r11d }, - .{ .imm = .u(0x10000000) }, - }); - try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000"); - - try enc.encode(.mov, &.{ - .{ .reg = .r11w }, - .{ .imm = .u(0x1000) }, - }); - try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000"); - - try enc.encode(.mov, &.{ - .{ .reg = .r11b }, - .{ .imm = .u(0x10) }, - }); - try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10"); -} - -test "lower FD/TD encoding" { - var enc = TestEncode{}; - - try enc.encode(.mov, &.{ - .{ .reg = .rax }, - .{ .mem = Instruction.Memory.initMoffs(.cs, 0x10) }, - }); - try expectEqualHexStrings("\x2E\x48\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs rax, cs:0x10"); - - try enc.encode(.mov, &.{ - .{ .reg = .eax }, - .{ .mem = Instruction.Memory.initMoffs(.fs, 0x10) }, - }); - try expectEqualHexStrings("\x64\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs eax, fs:0x10"); - - try enc.encode(.mov, &.{ - .{ .reg = .ax }, - .{ .mem = Instruction.Memory.initMoffs(.gs, 0x10) }, - }); - try expectEqualHexStrings("\x65\x66\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ax, gs:0x10"); - - try enc.encode(.mov, &.{ - .{ .reg = .al }, - .{ .mem = Instruction.Memory.initMoffs(.ds, 0x10) }, - }); - try expectEqualHexStrings("\xA0\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs al, ds:0x10"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initMoffs(.cs, 0x10) }, - .{ .reg = .rax }, - }); - try expectEqualHexStrings("\x2E\x48\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs cs:0x10, rax"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initMoffs(.fs, 0x10) }, - .{ .reg = .eax }, - }); - try expectEqualHexStrings("\x64\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs fs:0x10, eax"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initMoffs(.gs, 0x10) }, - .{ .reg = .ax }, - }); - try expectEqualHexStrings("\x65\x66\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs gs:0x10, ax"); - - try enc.encode(.mov, &.{ - .{ .mem = Instruction.Memory.initMoffs(.ds, 0x10) }, - .{ .reg = .al }, - }); - try expectEqualHexStrings("\xA2\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ds:0x10, al"); -} - -test "lower NP encoding" { - var enc = TestEncode{}; - - try enc.encode(.int3, &.{}); - try expectEqualHexStrings("\xCC", enc.code(), "int3"); - - try enc.encode(.nop, &.{}); - try expectEqualHexStrings("\x90", enc.code(), "nop"); - - try enc.encode(.ret, &.{}); - try expectEqualHexStrings("\xC3", enc.code(), "ret"); - - try enc.encode(.syscall, &.{}); - try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall"); -} - -fn invalidInstruction(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void { - const err: Instruction = .new(.none, mnemonic, ops); - try testing.expectError(error.InvalidInstruction, err); -} - -test "invalid instruction" { - try invalidInstruction(.call, &.{ - .{ .reg = .eax }, - }); - try invalidInstruction(.call, &.{ - .{ .reg = .ax }, - }); - try invalidInstruction(.call, &.{ - .{ .reg = .al }, - }); - try invalidInstruction(.call, &.{ - .{ .mem = Instruction.Memory.initRip(.dword, 0) }, - }); - try invalidInstruction(.call, &.{ - .{ .mem = Instruction.Memory.initRip(.word, 0) }, - }); - try invalidInstruction(.call, &.{ - .{ .mem = Instruction.Memory.initRip(.byte, 0) }, - }); - try invalidInstruction(.mov, &.{ - .{ .mem = Instruction.Memory.initRip(.word, 0x10) }, - .{ .reg = .r12 }, - }); - try invalidInstruction(.lea, &.{ - .{ .reg = .rax }, - .{ .reg = .rbx }, - }); - try invalidInstruction(.lea, &.{ - .{ .reg = .al }, - .{ .mem = Instruction.Memory.initRip(.byte, 0) }, - }); - try invalidInstruction(.pop, &.{ - .{ .reg = .r12b }, - }); - try invalidInstruction(.pop, &.{ - .{ .reg = .r12d }, - }); - try invalidInstruction(.push, &.{ - .{ .reg = .r12b }, - }); - try invalidInstruction(.push, &.{ - .{ .reg = .r12d }, - }); - try invalidInstruction(.push, &.{ - .{ .imm = .u(0x1000000000000000) }, - }); -} - -fn cannotEncode(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void { - try testing.expectError(error.CannotEncode, .new(.none, mnemonic, ops)); -} - -test "cannot encode" { - try cannotEncode(.@"test", &.{ - .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .r12 } }) }, - .{ .reg = .ah }, - }); - try cannotEncode(.@"test", &.{ - .{ .reg = .r11b }, - .{ .reg = .bh }, - }); - try cannotEncode(.mov, &.{ - .{ .reg = .sil }, - .{ .reg = .ah }, - }); -} - -const Assembler = struct { - it: Tokenizer, - - const Tokenizer = struct { - input: []const u8, - pos: usize = 0, - - const Error = error{InvalidToken}; - - const Token = struct { - id: Id, - start: usize, - end: usize, - - const Id = enum { - eof, - - space, - new_line, - - colon, - comma, - open_br, - close_br, - plus, - minus, - star, - - string, - numeral, - }; - }; - - const Iterator = struct {}; - - fn next(it: *Tokenizer) !Token { - var result = Token{ - .id = .eof, - .start = it.pos, - .end = it.pos, - }; - - var state: enum { - start, - space, - new_line, - string, - numeral, - numeral_hex, - } = .start; - - while (it.pos < it.input.len) : (it.pos += 1) { - const ch = it.input[it.pos]; - switch (state) { - .start => switch (ch) { - ',' => { - result.id = .comma; - it.pos += 1; - break; - }, - ':' => { - result.id = .colon; - it.pos += 1; - break; - }, - '[' => { - result.id = .open_br; - it.pos += 1; - break; - }, - ']' => { - result.id = .close_br; - it.pos += 1; - break; - }, - '+' => { - result.id = .plus; - it.pos += 1; - break; - }, - '-' => { - result.id = .minus; - it.pos += 1; - break; - }, - '*' => { - result.id = .star; - it.pos += 1; - break; - }, - ' ', '\t' => state = .space, - '\n', '\r' => state = .new_line, - 'a'...'z', 'A'...'Z' => state = .string, - '0'...'9' => state = .numeral, - else => return error.InvalidToken, - }, - - .space => switch (ch) { - ' ', '\t' => {}, - else => { - result.id = .space; - break; - }, - }, - - .new_line => switch (ch) { - '\n', '\r', ' ', '\t' => {}, - else => { - result.id = .new_line; - break; - }, - }, - - .string => switch (ch) { - 'a'...'z', 'A'...'Z', '0'...'9' => {}, - else => { - result.id = .string; - break; - }, - }, - - .numeral => switch (ch) { - 'x' => state = .numeral_hex, - '0'...'9' => {}, - else => { - result.id = .numeral; - break; - }, - }, - - .numeral_hex => switch (ch) { - 'a'...'f' => {}, - '0'...'9' => {}, - else => { - result.id = .numeral; - break; - }, - }, - } - } - - if (it.pos >= it.input.len) { - switch (state) { - .string => result.id = .string, - .numeral, .numeral_hex => result.id = .numeral, - else => {}, - } - } - - result.end = it.pos; - return result; - } - - fn seekTo(it: *Tokenizer, pos: usize) void { - it.pos = pos; - } - }; - - pub fn init(input: []const u8) Assembler { - return .{ - .it = Tokenizer{ .input = input }, - }; - } - - pub fn assemble(as: *Assembler, w: *Writer) !void { - while (try as.next()) |parsed_inst| { - const inst: Instruction = try .new(.none, parsed_inst.mnemonic, &parsed_inst.ops); - try inst.encode(w, .{}); - } - } - - const ParseResult = struct { - mnemonic: Instruction.Mnemonic, - ops: [4]Instruction.Operand, - }; - - const ParseError = error{ - UnexpectedToken, - InvalidMnemonic, - InvalidOperand, - InvalidRegister, - InvalidPtrSize, - InvalidMemoryOperand, - InvalidScaleIndex, - } || Tokenizer.Error || std.fmt.ParseIntError; - - fn next(as: *Assembler) ParseError!?ParseResult { - try as.skip(2, .{ .space, .new_line }); - const mnemonic_tok = as.expect(.string) catch |err| switch (err) { - error.UnexpectedToken => return if (try as.peek() == .eof) null else err, - else => return err, - }; - const mnemonic = mnemonicFromString(as.source(mnemonic_tok)) orelse - return error.InvalidMnemonic; - try as.skip(1, .{.space}); - - const rules = .{ - .{}, - .{.register}, - .{.memory}, - .{.immediate}, - .{ .register, .register }, - .{ .register, .memory }, - .{ .memory, .register }, - .{ .register, .immediate }, - .{ .memory, .immediate }, - .{ .register, .register, .immediate }, - .{ .register, .memory, .immediate }, - }; - - const pos = as.it.pos; - inline for (rules) |rule| { - var ops = [4]Instruction.Operand{ .none, .none, .none, .none }; - if (as.parseOperandRule(rule, &ops)) { - return .{ - .mnemonic = mnemonic, - .ops = ops, - }; - } else |_| { - as.it.seekTo(pos); - } - } - - return error.InvalidOperand; - } - - fn source(as: *Assembler, token: Tokenizer.Token) []const u8 { - return as.it.input[token.start..token.end]; - } - - fn peek(as: *Assembler) Tokenizer.Error!Tokenizer.Token.Id { - const pos = as.it.pos; - const next_tok = try as.it.next(); - const id = next_tok.id; - as.it.seekTo(pos); - return id; - } - - fn expect(as: *Assembler, id: Tokenizer.Token.Id) ParseError!Tokenizer.Token { - const next_tok_id = try as.peek(); - if (next_tok_id == id) return as.it.next(); - return error.UnexpectedToken; - } - - fn skip(as: *Assembler, comptime num: comptime_int, tok_ids: [num]Tokenizer.Token.Id) Tokenizer.Error!void { - outer: while (true) { - const pos = as.it.pos; - const next_tok = try as.it.next(); - inline for (tok_ids) |tok_id| { - if (next_tok.id == tok_id) continue :outer; - } - as.it.seekTo(pos); - break; - } - } - - fn mnemonicFromString(bytes: []const u8) ?Instruction.Mnemonic { - const ti = @typeInfo(Instruction.Mnemonic).@"enum"; - inline for (ti.fields) |field| { - if (std.mem.eql(u8, bytes, field.name)) { - return @field(Instruction.Mnemonic, field.name); - } - } - return null; - } - - fn parseOperandRule(as: *Assembler, rule: anytype, ops: *[4]Instruction.Operand) ParseError!void { - inline for (rule, 0..) |cond, i| { - comptime assert(i < 4); - if (i > 0) { - _ = try as.expect(.comma); - try as.skip(1, .{.space}); - } - if (@typeInfo(@TypeOf(cond)) != .enum_literal) { - @compileError("invalid condition in the rule: " ++ @typeName(@TypeOf(cond))); - } - switch (cond) { - .register => { - const reg_tok = try as.expect(.string); - const reg = registerFromString(as.source(reg_tok)) orelse - return error.InvalidOperand; - ops[i] = .{ .reg = reg }; - }, - .memory => { - const mem = try as.parseMemory(); - ops[i] = .{ .mem = mem }; - }, - .immediate => { - const is_neg = if (as.expect(.minus)) |_| true else |_| false; - const imm_tok = try as.expect(.numeral); - const imm: Instruction.Immediate = if (is_neg) blk: { - const imm = try std.fmt.parseInt(i32, as.source(imm_tok), 0); - break :blk .{ .signed = imm * -1 }; - } else .{ .unsigned = try std.fmt.parseInt(u64, as.source(imm_tok), 0) }; - ops[i] = .{ .imm = imm }; - }, - else => @compileError("unhandled enum literal " ++ @tagName(cond)), - } - try as.skip(1, .{.space}); - } - - try as.skip(1, .{.space}); - const tok = try as.it.next(); - switch (tok.id) { - .new_line, .eof => {}, - else => return error.InvalidOperand, - } - } - - fn registerFromString(bytes: []const u8) ?Register { - const ti = @typeInfo(Register).@"enum"; - inline for (ti.fields) |field| { - if (std.mem.eql(u8, bytes, field.name)) { - return @field(Register, field.name); - } - } - return null; - } - - fn parseMemory(as: *Assembler) ParseError!Instruction.Memory { - const ptr_size: ?Instruction.Memory.PtrSize = blk: { - const pos = as.it.pos; - const ptr_size = as.parsePtrSize() catch |err| switch (err) { - error.UnexpectedToken => { - as.it.seekTo(pos); - break :blk null; - }, - else => return err, - }; - break :blk ptr_size; - }; - - try as.skip(1, .{.space}); - - // Supported rules and orderings. - const rules = .{ - .{ .open_br, .general_purpose, .close_br }, // [ general_purpose ] - .{ .open_br, .general_purpose, .plus, .disp, .close_br }, // [ general_purpose + disp ] - .{ .open_br, .general_purpose, .minus, .disp, .close_br }, // [ general_purpose - disp ] - .{ .open_br, .disp, .plus, .general_purpose, .close_br }, // [ disp + general_purpose ] - .{ .open_br, .general_purpose, .plus, .index, .close_br }, // [ general_purpose + index ] - .{ .open_br, .general_purpose, .plus, .index, .star, .scale, .close_br }, // [ general_purpose + index * scale ] - .{ .open_br, .index, .star, .scale, .plus, .general_purpose, .close_br }, // [ index * scale + general_purpose ] - .{ .open_br, .general_purpose, .plus, .index, .star, .scale, .plus, .disp, .close_br }, // [ general_purpose + index * scale + disp ] - .{ .open_br, .general_purpose, .plus, .index, .star, .scale, .minus, .disp, .close_br }, // [ general_purpose + index * scale - disp ] - .{ .open_br, .index, .star, .scale, .plus, .general_purpose, .plus, .disp, .close_br }, // [ index * scale + general_purpose + disp ] - .{ .open_br, .index, .star, .scale, .plus, .general_purpose, .minus, .disp, .close_br }, // [ index * scale + general_purpose - disp ] - .{ .open_br, .disp, .plus, .index, .star, .scale, .plus, .general_purpose, .close_br }, // [ disp + index * scale + general_purpose ] - .{ .open_br, .disp, .plus, .general_purpose, .plus, .index, .star, .scale, .close_br }, // [ disp + general_purpose + index * scale ] - .{ .open_br, .general_purpose, .plus, .disp, .plus, .index, .star, .scale, .close_br }, // [ general_purpose + disp + index * scale ] - .{ .open_br, .general_purpose, .minus, .disp, .plus, .index, .star, .scale, .close_br }, // [ general_purpose - disp + index * scale ] - .{ .open_br, .general_purpose, .plus, .disp, .plus, .scale, .star, .index, .close_br }, // [ general_purpose + disp + scale * index ] - .{ .open_br, .general_purpose, .minus, .disp, .plus, .scale, .star, .index, .close_br }, // [ general_purpose - disp + scale * index ] - .{ .open_br, .rip, .plus, .disp, .close_br }, // [ rip + disp ] - .{ .open_br, .rip, .minus, .disp, .close_br }, // [ rig - disp ] - .{ .segment, .colon, .disp }, // seg:disp - }; - - const pos = as.it.pos; - inline for (rules) |rule| { - if (as.parseMemoryRule(rule)) |res| { - if (res.rip) { - if (res.base != null or res.scale_index != null or res.offset != null) - return error.InvalidMemoryOperand; - return Instruction.Memory.initRip(ptr_size orelse .qword, res.disp orelse 0); - } - if (res.base) |base| { - if (res.rip) - return error.InvalidMemoryOperand; - if (res.offset) |offset| { - if (res.scale_index != null or res.disp != null) - return error.InvalidMemoryOperand; - return Instruction.Memory.initMoffs(base, offset); - } - return Instruction.Memory.initSib(ptr_size orelse .qword, .{ - .base = .{ .reg = base }, - .scale_index = res.scale_index, - .disp = res.disp orelse 0, - }); - } - return error.InvalidMemoryOperand; - } else |_| { - as.it.seekTo(pos); - } - } - - return error.InvalidOperand; - } - - const MemoryParseResult = struct { - rip: bool = false, - base: ?Register = null, - scale_index: ?Instruction.Memory.ScaleIndex = null, - disp: ?i32 = null, - offset: ?u64 = null, - }; - - fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!MemoryParseResult { - var res: MemoryParseResult = .{}; - inline for (rule, 0..) |cond, i| { - if (@typeInfo(@TypeOf(cond)) != .enum_literal) { - @compileError("unsupported condition type in the rule: " ++ @typeName(@TypeOf(cond))); - } - switch (cond) { - .open_br, .close_br, .plus, .minus, .star, .colon => { - _ = try as.expect(cond); - }, - .general_purpose, .segment => { - const tok = try as.expect(.string); - const base = registerFromString(as.source(tok)) orelse return error.InvalidMemoryOperand; - if (!base.isClass(cond)) return error.InvalidMemoryOperand; - res.base = base; - }, - .rip => { - const tok = try as.expect(.string); - if (!std.mem.eql(u8, as.source(tok), "rip")) return error.InvalidMemoryOperand; - res.rip = true; - }, - .index => { - const tok = try as.expect(.string); - const index = registerFromString(as.source(tok)) orelse - return error.InvalidMemoryOperand; - if (res.scale_index) |*si| { - si.index = index; - } else { - res.scale_index = .{ .scale = 1, .index = index }; - } - }, - .scale => { - const tok = try as.expect(.numeral); - const scale = try std.fmt.parseInt(u2, as.source(tok), 0); - if (res.scale_index) |*si| { - si.scale = scale; - } else { - res.scale_index = .{ .scale = scale, .index = undefined }; - } - }, - .disp => { - const tok = try as.expect(.numeral); - const is_neg = blk: { - if (i > 0) { - if (rule[i - 1] == .minus) break :blk true; - } - break :blk false; - }; - if (std.fmt.parseInt(i32, as.source(tok), 0)) |disp| { - res.disp = if (is_neg) -1 * disp else disp; - } else |err| switch (err) { - error.Overflow => { - if (is_neg) return err; - if (res.base) |base| { - if (!base.isClass(.segment)) return err; - } - const offset = try std.fmt.parseInt(u64, as.source(tok), 0); - res.offset = offset; - }, - else => return err, - } - }, - else => @compileError("unhandled operand output type: " ++ @tagName(cond)), - } - try as.skip(1, .{.space}); - } - return res; - } - - fn parsePtrSize(as: *Assembler) ParseError!Instruction.Memory.PtrSize { - const size = try as.expect(.string); - try as.skip(1, .{.space}); - const ptr = try as.expect(.string); - - const size_raw = as.source(size); - const ptr_raw = as.source(ptr); - const len = size_raw.len + ptr_raw.len + 1; - var buf: ["qword ptr".len]u8 = undefined; - if (len > buf.len) return error.InvalidPtrSize; - - for (size_raw, 0..) |c, i| { - buf[i] = std.ascii.toLower(c); - } - buf[size_raw.len] = ' '; - for (ptr_raw, 0..) |c, i| { - buf[size_raw.len + i + 1] = std.ascii.toLower(c); - } - - const slice = buf[0..len]; - if (std.mem.eql(u8, slice, "qword ptr")) return .qword; - if (std.mem.eql(u8, slice, "dword ptr")) return .dword; - if (std.mem.eql(u8, slice, "word ptr")) return .word; - if (std.mem.eql(u8, slice, "byte ptr")) return .byte; - if (std.mem.eql(u8, slice, "tbyte ptr")) return .tbyte; - return error.InvalidPtrSize; - } -}; - -test "assemble" { - const input = - \\int3 - \\mov rax, rbx - \\mov qword ptr [rbp], rax - \\mov qword ptr [rbp - 16], rax - \\mov qword ptr [16 + rbp], rax - \\mov rax, 0x10 - \\mov byte ptr [rbp - 0x10], 0x10 - \\mov word ptr [rbp + r12], r11w - \\mov word ptr [rbp + r12 * 2], r11w - \\mov word ptr [rbp + r12 * 2 - 16], r11w - \\mov dword ptr [rip - 16], r12d - \\mov rax, fs:0x0 - \\mov rax, gs:0x1000000000000000 - \\movzx r12, al - \\imul r12, qword ptr [rbp - 16], 6 - \\jmp 0x0 - \\jc 0x0 - \\jb 0x0 - \\sal rax, 1 - \\sal rax, 63 - \\shl rax, 63 - \\sar rax, 63 - \\shr rax, 63 - \\test byte ptr [rbp - 16], r12b - \\sal r12, cl - \\mul qword ptr [rip - 16] - \\div r12 - \\idiv byte ptr [rbp - 16] - \\cwde - \\cbw - \\cdqe - \\test byte ptr [rbp], ah - \\test byte ptr [r12], spl - \\cdq - \\cwd - \\cqo - \\test bl, 0x1 - \\mov rbx,0x8000000000000000 - \\movss xmm0, dword ptr [rbp] - \\movss xmm0, xmm1 - \\movss dword ptr [rbp - 16 + rax * 2], xmm7 - \\movss dword ptr [rbp - 16 + rax * 2], xmm8 - \\movss xmm15, xmm9 - \\movsd xmm8, qword ptr [rbp - 16] - \\movsd qword ptr [rbp - 8], xmm0 - \\movq xmm8, qword ptr [rbp - 16] - \\movq qword ptr [rbp - 16], xmm8 - \\ucomisd xmm0, qword ptr [rbp - 16] - \\fisttp qword ptr [rbp - 16] - \\fisttp word ptr [rip + 32] - \\fisttp dword ptr [rax] - \\fld tbyte ptr [rbp] - \\fld dword ptr [rbp] - \\xor bl, 0xff - \\ud2 - \\add rsp, -1 - \\add rsp, 0xff - \\mov sil, byte ptr [rax + rcx * 1] - \\ - ; - - // zig fmt: off - const expected = &[_]u8{ - 0xCC, - 0x48, 0x89, 0xD8, - 0x48, 0x89, 0x45, 0x00, - 0x48, 0x89, 0x45, 0xF0, - 0x48, 0x89, 0x45, 0x10, - 0x48, 0xC7, 0xC0, 0x10, 0x00, 0x00, 0x00, - 0xC6, 0x45, 0xF0, 0x10, - 0x66, 0x46, 0x89, 0x5C, 0x25, 0x00, - 0x66, 0x46, 0x89, 0x5C, 0x65, 0x00, - 0x66, 0x46, 0x89, 0x5C, 0x65, 0xF0, - 0x44, 0x89, 0x25, 0xF0, 0xFF, 0xFF, 0xFF, - 0x64, 0x48, 0x8B, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, - 0x65, 0x48, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, - 0x4C, 0x0F, 0xB6, 0xE0, - 0x4C, 0x6B, 0x65, 0xF0, 0x06, - 0xE9, 0x00, 0x00, 0x00, 0x00, - 0x0F, 0x82, 0x00, 0x00, 0x00, 0x00, - 0x0F, 0x82, 0x00, 0x00, 0x00, 0x00, - 0x48, 0xD1, 0xE0, - 0x48, 0xC1, 0xE0, 0x3F, - 0x48, 0xC1, 0xE0, 0x3F, - 0x48, 0xC1, 0xF8, 0x3F, - 0x48, 0xC1, 0xE8, 0x3F, - 0x44, 0x84, 0x65, 0xF0, - 0x49, 0xD3, 0xE4, - 0x48, 0xF7, 0x25, 0xF0, 0xFF, 0xFF, 0xFF, - 0x49, 0xF7, 0xF4, - 0xF6, 0x7D, 0xF0, - 0x98, - 0x66, 0x98, - 0x48, 0x98, - 0x84, 0x65, 0x00, - 0x41, 0x84, 0x24, 0x24, - 0x99, - 0x66, 0x99, - 0x48, 0x99, - 0xF6, 0xC3, 0x01, - 0x48, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0xF3, 0x0F, 0x10, 0x45, 0x00, - 0xF3, 0x0F, 0x10, 0xC1, - 0xF3, 0x0F, 0x11, 0x7C, 0x45, 0xF0, - 0xF3, 0x44, 0x0F, 0x11, 0x44, 0x45, 0xF0, - 0xF3, 0x45, 0x0F, 0x10, 0xF9, - 0xF2, 0x44, 0x0F, 0x10, 0x45, 0xF0, - 0xF2, 0x0F, 0x11, 0x45, 0xF8, - 0x66, 0x4C, 0x0F, 0x6E, 0x45, 0xF0, - 0x66, 0x4C, 0x0F, 0x7E, 0x45, 0xF0, - 0x66, 0x0F, 0x2E, 0x45, 0xF0, - 0xDD, 0x4D, 0xF0, - 0xDF, 0x0D, 0x20, 0x00, 0x00, 0x00, - 0xDB, 0x08, - 0xDB, 0x6D, 0x00, - 0xD9, 0x45, 0x00, - 0x80, 0xF3, 0xFF, - 0x0F, 0x0B, - 0x48, 0x83, 0xC4, 0xFF, - 0x48, 0x81, 0xC4, 0xFF, 0x00, 0x00, 0x00, - 0x40, 0x8A, 0x34, 0x08, - }; - // zig fmt: on - - var as = Assembler.init(input); - var output = std.array_list.Managed(u8).init(testing.allocator); - defer output.deinit(); - try as.assemble(output.writer()); - try expectEqualHexStrings(expected, output.items, input); -} - -test "assemble - Jcc" { - const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{ - .{ .ja, 0x87 }, - .{ .jae, 0x83 }, - .{ .jb, 0x82 }, - .{ .jbe, 0x86 }, - .{ .jc, 0x82 }, - .{ .je, 0x84 }, - .{ .jg, 0x8f }, - .{ .jge, 0x8d }, - .{ .jl, 0x8c }, - .{ .jle, 0x8e }, - .{ .jna, 0x86 }, - .{ .jnae, 0x82 }, - .{ .jnb, 0x83 }, - .{ .jnbe, 0x87 }, - .{ .jnc, 0x83 }, - .{ .jne, 0x85 }, - .{ .jng, 0x8e }, - .{ .jnge, 0x8c }, - .{ .jnl, 0x8d }, - .{ .jnle, 0x8f }, - .{ .jno, 0x81 }, - .{ .jnp, 0x8b }, - .{ .jns, 0x89 }, - .{ .jnz, 0x85 }, - .{ .jo, 0x80 }, - .{ .jp, 0x8a }, - .{ .jpe, 0x8a }, - .{ .jpo, 0x8b }, - .{ .js, 0x88 }, - .{ .jz, 0x84 }, - }; - - inline for (&mnemonics) |mnemonic| { - const input = @tagName(mnemonic[0]) ++ " 0x0"; - const expected = [_]u8{ 0x0f, mnemonic[1], 0x0, 0x0, 0x0, 0x0 }; - var as = Assembler.init(input); - var output = std.array_list.Managed(u8).init(testing.allocator); - defer output.deinit(); - try as.assemble(output.writer()); - try expectEqualHexStrings(&expected, output.items, input); - } -} - -test "assemble - SETcc" { - const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{ - .{ .seta, 0x97 }, - .{ .setae, 0x93 }, - .{ .setb, 0x92 }, - .{ .setbe, 0x96 }, - .{ .setc, 0x92 }, - .{ .sete, 0x94 }, - .{ .setg, 0x9f }, - .{ .setge, 0x9d }, - .{ .setl, 0x9c }, - .{ .setle, 0x9e }, - .{ .setna, 0x96 }, - .{ .setnae, 0x92 }, - .{ .setnb, 0x93 }, - .{ .setnbe, 0x97 }, - .{ .setnc, 0x93 }, - .{ .setne, 0x95 }, - .{ .setng, 0x9e }, - .{ .setnge, 0x9c }, - .{ .setnl, 0x9d }, - .{ .setnle, 0x9f }, - .{ .setno, 0x91 }, - .{ .setnp, 0x9b }, - .{ .setns, 0x99 }, - .{ .setnz, 0x95 }, - .{ .seto, 0x90 }, - .{ .setp, 0x9a }, - .{ .setpe, 0x9a }, - .{ .setpo, 0x9b }, - .{ .sets, 0x98 }, - .{ .setz, 0x94 }, - }; - - inline for (&mnemonics) |mnemonic| { - const input = @tagName(mnemonic[0]) ++ " al"; - const expected = [_]u8{ 0x0f, mnemonic[1], 0xC0 }; - var as = Assembler.init(input); - var output = std.array_list.Managed(u8).init(testing.allocator); - defer output.deinit(); - try as.assemble(output.writer()); - try expectEqualHexStrings(&expected, output.items, input); - } -} - -test "assemble - CMOVcc" { - const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{ - .{ .cmova, 0x47 }, - .{ .cmovae, 0x43 }, - .{ .cmovb, 0x42 }, - .{ .cmovbe, 0x46 }, - .{ .cmovc, 0x42 }, - .{ .cmove, 0x44 }, - .{ .cmovg, 0x4f }, - .{ .cmovge, 0x4d }, - .{ .cmovl, 0x4c }, - .{ .cmovle, 0x4e }, - .{ .cmovna, 0x46 }, - .{ .cmovnae, 0x42 }, - .{ .cmovnb, 0x43 }, - .{ .cmovnbe, 0x47 }, - .{ .cmovnc, 0x43 }, - .{ .cmovne, 0x45 }, - .{ .cmovng, 0x4e }, - .{ .cmovnge, 0x4c }, - .{ .cmovnl, 0x4d }, - .{ .cmovnle, 0x4f }, - .{ .cmovno, 0x41 }, - .{ .cmovnp, 0x4b }, - .{ .cmovns, 0x49 }, - .{ .cmovnz, 0x45 }, - .{ .cmovo, 0x40 }, - .{ .cmovp, 0x4a }, - .{ .cmovpe, 0x4a }, - .{ .cmovpo, 0x4b }, - .{ .cmovs, 0x48 }, - .{ .cmovz, 0x44 }, - }; - - inline for (&mnemonics) |mnemonic| { - const input = @tagName(mnemonic[0]) ++ " rax, rbx"; - const expected = [_]u8{ 0x48, 0x0f, mnemonic[1], 0xC3 }; - var as = Assembler.init(input); - var output = std.array_list.Managed(u8).init(testing.allocator); - defer output.deinit(); - try as.assemble(output.writer()); - try expectEqualHexStrings(&expected, output.items, input); - } -} diff --git a/src/arch/x86_64/encodings.zon b/src/arch/x86_64/encodings.zon deleted file mode 100644 index ea136569ce939d97982931afef4c5ae54a07322a..0000000000000000000000000000000000000000 --- a/src/arch/x86_64/encodings.zon +++ /dev/null @@ -1,2700 +0,0 @@ -// zig fmt: off -.{ - // General-purpose - .{ .aaa, .z, .{}, .{ 0x37 }, 0, .none, .@"32bit" }, - - .{ .aad, .z, .{ }, .{ 0xd5, 0x0a }, 0, .none, .@"32bit" }, - .{ .aad, .zi, .{ .imm8 }, .{ 0xd5 }, 0, .none, .@"32bit" }, - - .{ .aam, .z, .{ }, .{ 0xd4, 0x0a }, 0, .none, .@"32bit" }, - .{ .aam, .zi, .{ .imm8 }, .{ 0xd4 }, 0, .none, .@"32bit" }, - - .{ .aas, .z, .{}, .{ 0x3f }, 0, .none, .@"32bit" }, - - .{ .adc, .zi, .{ .al, .imm8 }, .{ 0x14 }, 0, .none, .none }, - .{ .adc, .zi, .{ .ax, .imm16 }, .{ 0x15 }, 0, .short, .none }, - .{ .adc, .zi, .{ .eax, .imm32 }, .{ 0x15 }, 0, .none, .none }, - .{ .adc, .zi, .{ .rax, .imm32s }, .{ 0x15 }, 0, .long, .none }, - .{ .adc, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 2, .none, .none }, - .{ .adc, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 2, .rex, .none }, - .{ .adc, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 2, .short, .none }, - .{ .adc, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 2, .none, .none }, - .{ .adc, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 2, .long, .none }, - .{ .adc, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 2, .short, .none }, - .{ .adc, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 2, .none, .none }, - .{ .adc, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 2, .long, .none }, - .{ .adc, .mr, .{ .rm8, .r8 }, .{ 0x10 }, 0, .none, .none }, - .{ .adc, .mr, .{ .rm8, .r8 }, .{ 0x10 }, 0, .rex, .none }, - .{ .adc, .mr, .{ .rm16, .r16 }, .{ 0x11 }, 0, .short, .none }, - .{ .adc, .mr, .{ .rm32, .r32 }, .{ 0x11 }, 0, .none, .none }, - .{ .adc, .mr, .{ .rm64, .r64 }, .{ 0x11 }, 0, .long, .none }, - .{ .adc, .rm, .{ .r8, .rm8 }, .{ 0x12 }, 0, .none, .none }, - .{ .adc, .rm, .{ .r8, .rm8 }, .{ 0x12 }, 0, .rex, .none }, - .{ .adc, .rm, .{ .r16, .rm16 }, .{ 0x13 }, 0, .short, .none }, - .{ .adc, .rm, .{ .r32, .rm32 }, .{ 0x13 }, 0, .none, .none }, - .{ .adc, .rm, .{ .r64, .rm64 }, .{ 0x13 }, 0, .long, .none }, - - .{ .add, .zi, .{ .al, .imm8 }, .{ 0x04 }, 0, .none, .none }, - .{ .add, .zi, .{ .ax, .imm16 }, .{ 0x05 }, 0, .short, .none }, - .{ .add, .zi, .{ .eax, .imm32 }, .{ 0x05 }, 0, .none, .none }, - .{ .add, .zi, .{ .rax, .imm32s }, .{ 0x05 }, 0, .long, .none }, - .{ .add, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 0, .none, .none }, - .{ .add, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 0, .rex, .none }, - .{ .add, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 0, .short, .none }, - .{ .add, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 0, .none, .none }, - .{ .add, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 0, .long, .none }, - .{ .add, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 0, .short, .none }, - .{ .add, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 0, .none, .none }, - .{ .add, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 0, .long, .none }, - .{ .add, .mr, .{ .rm8, .r8 }, .{ 0x00 }, 0, .none, .none }, - .{ .add, .mr, .{ .rm8, .r8 }, .{ 0x00 }, 0, .rex, .none }, - .{ .add, .mr, .{ .rm16, .r16 }, .{ 0x01 }, 0, .short, .none }, - .{ .add, .mr, .{ .rm32, .r32 }, .{ 0x01 }, 0, .none, .none }, - .{ .add, .mr, .{ .rm64, .r64 }, .{ 0x01 }, 0, .long, .none }, - .{ .add, .rm, .{ .r8, .rm8 }, .{ 0x02 }, 0, .none, .none }, - .{ .add, .rm, .{ .r8, .rm8 }, .{ 0x02 }, 0, .rex, .none }, - .{ .add, .rm, .{ .r16, .rm16 }, .{ 0x03 }, 0, .short, .none }, - .{ .add, .rm, .{ .r32, .rm32 }, .{ 0x03 }, 0, .none, .none }, - .{ .add, .rm, .{ .r64, .rm64 }, .{ 0x03 }, 0, .long, .none }, - - .{ .@"and", .zi, .{ .al, .imm8 }, .{ 0x24 }, 0, .none, .none }, - .{ .@"and", .zi, .{ .ax, .imm16 }, .{ 0x25 }, 0, .short, .none }, - .{ .@"and", .zi, .{ .eax, .imm32 }, .{ 0x25 }, 0, .none, .none }, - .{ .@"and", .zi, .{ .rax, .imm32s }, .{ 0x25 }, 0, .long, .none }, - .{ .@"and", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 4, .none, .none }, - .{ .@"and", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 4, .rex, .none }, - .{ .@"and", .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 4, .short, .none }, - .{ .@"and", .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 4, .none, .none }, - .{ .@"and", .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 4, .long, .none }, - .{ .@"and", .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 4, .short, .none }, - .{ .@"and", .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 4, .none, .none }, - .{ .@"and", .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 4, .long, .none }, - .{ .@"and", .mr, .{ .rm8, .r8 }, .{ 0x20 }, 0, .none, .none }, - .{ .@"and", .mr, .{ .rm8, .r8 }, .{ 0x20 }, 0, .rex, .none }, - .{ .@"and", .mr, .{ .rm16, .r16 }, .{ 0x21 }, 0, .short, .none }, - .{ .@"and", .mr, .{ .rm32, .r32 }, .{ 0x21 }, 0, .none, .none }, - .{ .@"and", .mr, .{ .rm64, .r64 }, .{ 0x21 }, 0, .long, .none }, - .{ .@"and", .rm, .{ .r8, .rm8 }, .{ 0x22 }, 0, .none, .none }, - .{ .@"and", .rm, .{ .r8, .rm8 }, .{ 0x22 }, 0, .rex, .none }, - .{ .@"and", .rm, .{ .r16, .rm16 }, .{ 0x23 }, 0, .short, .none }, - .{ .@"and", .rm, .{ .r32, .rm32 }, .{ 0x23 }, 0, .none, .none }, - .{ .@"and", .rm, .{ .r64, .rm64 }, .{ 0x23 }, 0, .long, .none }, - - .{ .arpl, .mr, .{ .rm16, .r16 }, .{ 0x63 }, 0, .none, .@"32bit" }, - - .{ .bound, .rm, .{ .r16, .m }, .{ 0x62 }, 0, .short, .@"32bit" }, - .{ .bound, .rm, .{ .r32, .m }, .{ 0x62 }, 0, .short, .@"32bit" }, - - .{ .bsf, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0xbc }, 0, .short, .none }, - .{ .bsf, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xbc }, 0, .none, .none }, - .{ .bsf, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0xbc }, 0, .long, .none }, - - .{ .bsr, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0xbd }, 0, .short, .none }, - .{ .bsr, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xbd }, 0, .none, .none }, - .{ .bsr, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0xbd }, 0, .long, .none }, - - .{ .bswap, .o, .{ .r32 }, .{ 0x0f, 0xc8 }, 0, .none, .none }, - .{ .bswap, .o, .{ .r64 }, .{ 0x0f, 0xc8 }, 0, .long, .none }, - - .{ .bt, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xa3 }, 0, .short, .none }, - .{ .bt, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xa3 }, 0, .none, .none }, - .{ .bt, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xa3 }, 0, .long, .none }, - .{ .bt, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 4, .short, .none }, - .{ .bt, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 4, .none, .none }, - .{ .bt, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 4, .long, .none }, - - .{ .btc, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xbb }, 0, .short, .none }, - .{ .btc, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xbb }, 0, .none, .none }, - .{ .btc, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xbb }, 0, .long, .none }, - .{ .btc, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 7, .short, .none }, - .{ .btc, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 7, .none, .none }, - .{ .btc, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 7, .long, .none }, - - .{ .btr, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xb3 }, 0, .short, .none }, - .{ .btr, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xb3 }, 0, .none, .none }, - .{ .btr, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xb3 }, 0, .long, .none }, - .{ .btr, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 6, .short, .none }, - .{ .btr, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 6, .none, .none }, - .{ .btr, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 6, .long, .none }, - - .{ .bts, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xab }, 0, .short, .none }, - .{ .bts, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xab }, 0, .none, .none }, - .{ .bts, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xab }, 0, .long, .none }, - .{ .bts, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 5, .short, .none }, - .{ .bts, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 5, .none, .none }, - .{ .bts, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 5, .long, .none }, - - .{ .call, .d, .{ .rel32 }, .{ 0xe8 }, 0, .none, .none }, - .{ .call, .m, .{ .rm32 }, .{ 0xff }, 2, .none, .@"32bit" }, - .{ .call, .m, .{ .rm64 }, .{ 0xff }, 2, .none, .@"64bit" }, - - .{ .cbw, .z, .{}, .{ 0x98 }, 0, .short, .none }, - .{ .cwde, .z, .{}, .{ 0x98 }, 0, .none, .none }, - .{ .cdqe, .z, .{}, .{ 0x98 }, 0, .long, .none }, - - .{ .clac, .z, .{}, .{ 0x0f, 0x01, 0xca }, 0, .none, .smap }, - - .{ .clc, .z, .{}, .{ 0xf8 }, 0, .none, .none }, - - .{ .cld, .z, .{}, .{ 0xfc }, 0, .none, .none }, - - .{ .cldemote, .m, .{ .m8 }, .{ 0x0f, 0x1c }, 0, .none, .cldemote }, - - .{ .clflush, .m, .{ .m8 }, .{ 0x0f, 0xae }, 7, .none, .none }, - - .{ .clflushopt, .m, .{ .m8 }, .{ 0x66, 0x0f, 0xae }, 7, .none, .clflushopt }, - - .{ .cli, .z, .{}, .{ 0xfa }, 0, .none, .none }, - - .{ .clrssbsy, .m, .{ .m64 }, .{ 0xf3, 0x0f, 0xae }, 6, .none, .shstk }, - - .{ .clts, .z, .{}, .{ 0x0f, 0x06 }, 0, .none, .none }, - - .{ .clui, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr }, - - .{ .clwb, .m, .{ .m8 }, .{ 0x66, 0x0f, 0xae }, 6, .none, .clwb }, - - .{ .cmc, .z, .{}, .{ 0xf5 }, 0, .none, .none }, - - .{ .cmova, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x47 }, 0, .short, .cmov }, - .{ .cmova, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x47 }, 0, .none, .cmov }, - .{ .cmova, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x47 }, 0, .long, .cmov }, - .{ .cmovae, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x43 }, 0, .short, .cmov }, - .{ .cmovae, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x43 }, 0, .none, .cmov }, - .{ .cmovae, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x43 }, 0, .long, .cmov }, - .{ .cmovb, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x42 }, 0, .short, .cmov }, - .{ .cmovb, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x42 }, 0, .none, .cmov }, - .{ .cmovb, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x42 }, 0, .long, .cmov }, - .{ .cmovbe, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x46 }, 0, .short, .cmov }, - .{ .cmovbe, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x46 }, 0, .none, .cmov }, - .{ .cmovbe, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x46 }, 0, .long, .cmov }, - .{ .cmovc, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x42 }, 0, .short, .cmov }, - .{ .cmovc, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x42 }, 0, .none, .cmov }, - .{ .cmovc, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x42 }, 0, .long, .cmov }, - .{ .cmove, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x44 }, 0, .short, .cmov }, - .{ .cmove, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x44 }, 0, .none, .cmov }, - .{ .cmove, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x44 }, 0, .long, .cmov }, - .{ .cmovg, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4f }, 0, .short, .cmov }, - .{ .cmovg, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4f }, 0, .none, .cmov }, - .{ .cmovg, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4f }, 0, .long, .cmov }, - .{ .cmovge, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4d }, 0, .short, .cmov }, - .{ .cmovge, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4d }, 0, .none, .cmov }, - .{ .cmovge, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4d }, 0, .long, .cmov }, - .{ .cmovl, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4c }, 0, .short, .cmov }, - .{ .cmovl, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4c }, 0, .none, .cmov }, - .{ .cmovl, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4c }, 0, .long, .cmov }, - .{ .cmovle, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4e }, 0, .short, .cmov }, - .{ .cmovle, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4e }, 0, .none, .cmov }, - .{ .cmovle, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4e }, 0, .long, .cmov }, - .{ .cmovna, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x46 }, 0, .short, .cmov }, - .{ .cmovna, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x46 }, 0, .none, .cmov }, - .{ .cmovna, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x46 }, 0, .long, .cmov }, - .{ .cmovnae, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x42 }, 0, .short, .cmov }, - .{ .cmovnae, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x42 }, 0, .none, .cmov }, - .{ .cmovnae, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x42 }, 0, .long, .cmov }, - .{ .cmovnb, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x43 }, 0, .short, .cmov }, - .{ .cmovnb, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x43 }, 0, .none, .cmov }, - .{ .cmovnb, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x43 }, 0, .long, .cmov }, - .{ .cmovnbe, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x47 }, 0, .short, .cmov }, - .{ .cmovnbe, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x47 }, 0, .none, .cmov }, - .{ .cmovnbe, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x47 }, 0, .long, .cmov }, - .{ .cmovnc, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x43 }, 0, .short, .cmov }, - .{ .cmovnc, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x43 }, 0, .none, .cmov }, - .{ .cmovnc, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x43 }, 0, .long, .cmov }, - .{ .cmovne, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x45 }, 0, .short, .cmov }, - .{ .cmovne, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x45 }, 0, .none, .cmov }, - .{ .cmovne, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x45 }, 0, .long, .cmov }, - .{ .cmovng, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4e }, 0, .short, .cmov }, - .{ .cmovng, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4e }, 0, .none, .cmov }, - .{ .cmovng, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4e }, 0, .long, .cmov }, - .{ .cmovnge, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4c }, 0, .short, .cmov }, - .{ .cmovnge, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4c }, 0, .none, .cmov }, - .{ .cmovnge, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4c }, 0, .long, .cmov }, - .{ .cmovnl, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4d }, 0, .short, .cmov }, - .{ .cmovnl, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4d }, 0, .none, .cmov }, - .{ .cmovnl, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4d }, 0, .long, .cmov }, - .{ .cmovnle, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4f }, 0, .short, .cmov }, - .{ .cmovnle, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4f }, 0, .none, .cmov }, - .{ .cmovnle, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4f }, 0, .long, .cmov }, - .{ .cmovno, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x41 }, 0, .short, .cmov }, - .{ .cmovno, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x41 }, 0, .none, .cmov }, - .{ .cmovno, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x41 }, 0, .long, .cmov }, - .{ .cmovnp, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4b }, 0, .short, .cmov }, - .{ .cmovnp, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4b }, 0, .none, .cmov }, - .{ .cmovnp, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4b }, 0, .long, .cmov }, - .{ .cmovns, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x49 }, 0, .short, .cmov }, - .{ .cmovns, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x49 }, 0, .none, .cmov }, - .{ .cmovns, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x49 }, 0, .long, .cmov }, - .{ .cmovnz, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x45 }, 0, .short, .cmov }, - .{ .cmovnz, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x45 }, 0, .none, .cmov }, - .{ .cmovnz, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x45 }, 0, .long, .cmov }, - .{ .cmovo, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x40 }, 0, .short, .cmov }, - .{ .cmovo, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x40 }, 0, .none, .cmov }, - .{ .cmovo, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x40 }, 0, .long, .cmov }, - .{ .cmovp, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4a }, 0, .short, .cmov }, - .{ .cmovp, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4a }, 0, .none, .cmov }, - .{ .cmovp, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4a }, 0, .long, .cmov }, - .{ .cmovpe, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4a }, 0, .short, .cmov }, - .{ .cmovpe, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4a }, 0, .none, .cmov }, - .{ .cmovpe, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4a }, 0, .long, .cmov }, - .{ .cmovpo, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4b }, 0, .short, .cmov }, - .{ .cmovpo, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4b }, 0, .none, .cmov }, - .{ .cmovpo, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4b }, 0, .long, .cmov }, - .{ .cmovs, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x48 }, 0, .short, .cmov }, - .{ .cmovs, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x48 }, 0, .none, .cmov }, - .{ .cmovs, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x48 }, 0, .long, .cmov }, - .{ .cmovz, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x44 }, 0, .short, .cmov }, - .{ .cmovz, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x44 }, 0, .none, .cmov }, - .{ .cmovz, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x44 }, 0, .long, .cmov }, - - .{ .cmp, .zi, .{ .al, .imm8 }, .{ 0x3c }, 0, .none, .none }, - .{ .cmp, .zi, .{ .ax, .imm16 }, .{ 0x3d }, 0, .short, .none }, - .{ .cmp, .zi, .{ .eax, .imm32 }, .{ 0x3d }, 0, .none, .none }, - .{ .cmp, .zi, .{ .rax, .imm32s }, .{ 0x3d }, 0, .long, .none }, - .{ .cmp, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 7, .none, .none }, - .{ .cmp, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 7, .rex, .none }, - .{ .cmp, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 7, .short, .none }, - .{ .cmp, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 7, .none, .none }, - .{ .cmp, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 7, .long, .none }, - .{ .cmp, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 7, .short, .none }, - .{ .cmp, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 7, .none, .none }, - .{ .cmp, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 7, .long, .none }, - .{ .cmp, .mr, .{ .rm8, .r8 }, .{ 0x38 }, 0, .none, .none }, - .{ .cmp, .mr, .{ .rm8, .r8 }, .{ 0x38 }, 0, .rex, .none }, - .{ .cmp, .mr, .{ .rm16, .r16 }, .{ 0x39 }, 0, .short, .none }, - .{ .cmp, .mr, .{ .rm32, .r32 }, .{ 0x39 }, 0, .none, .none }, - .{ .cmp, .mr, .{ .rm64, .r64 }, .{ 0x39 }, 0, .long, .none }, - .{ .cmp, .rm, .{ .r8, .rm8 }, .{ 0x3a }, 0, .none, .none }, - .{ .cmp, .rm, .{ .r8, .rm8 }, .{ 0x3a }, 0, .rex, .none }, - .{ .cmp, .rm, .{ .r16, .rm16 }, .{ 0x3b }, 0, .short, .none }, - .{ .cmp, .rm, .{ .r32, .rm32 }, .{ 0x3b }, 0, .none, .none }, - .{ .cmp, .rm, .{ .r64, .rm64 }, .{ 0x3b }, 0, .long, .none }, - - .{ .cmps, .z, .{ .m8, .m8 }, .{ 0xa6 }, 0, .none, .none }, - .{ .cmps, .z, .{ .m16, .m16 }, .{ 0xa7 }, 0, .short, .none }, - .{ .cmps, .z, .{ .m32, .m32 }, .{ 0xa7 }, 0, .none, .none }, - .{ .cmps, .z, .{ .m64, .m64 }, .{ 0xa7 }, 0, .long, .none }, - .{ .cmpsb, .z, .{ }, .{ 0xa6 }, 0, .none, .none }, - .{ .cmpsw, .z, .{ }, .{ 0xa7 }, 0, .short, .none }, - .{ .cmpsd, .z, .{ }, .{ 0xa7 }, 0, .none, .none }, - .{ .cmpsq, .z, .{ }, .{ 0xa7 }, 0, .long, .none }, - - .{ .cmpxchg, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xb0 }, 0, .none, .none }, - .{ .cmpxchg, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xb0 }, 0, .rex, .none }, - .{ .cmpxchg, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xb1 }, 0, .short, .none }, - .{ .cmpxchg, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xb1 }, 0, .none, .none }, - .{ .cmpxchg, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xb1 }, 0, .long, .none }, - - .{ .cmpxchg8b, .m, .{ .m64 }, .{ 0x0f, 0xc7 }, 1, .none, .none }, - .{ .cmpxchg16b, .m, .{ .m128 }, .{ 0x0f, 0xc7 }, 1, .long, .none }, - - .{ .cpuid, .z, .{}, .{ 0x0f, 0xa2 }, 0, .none, .none }, - - .{ .cwd, .z, .{}, .{ 0x99 }, 0, .short, .none }, - .{ .cdq, .z, .{}, .{ 0x99 }, 0, .none, .none }, - .{ .cqo, .z, .{}, .{ 0x99 }, 0, .long, .none }, - - .{ .daa, .z, .{}, .{ 0x27 }, 0, .none, .@"32bit" }, - - .{ .das, .z, .{}, .{ 0x27 }, 0, .none, .@"32bit" }, - - .{ .dec, .m, .{ .rm8 }, .{ 0xfe }, 1, .none, .none }, - .{ .dec, .m, .{ .rm8 }, .{ 0xfe }, 1, .rex, .none }, - .{ .dec, .m, .{ .rm16 }, .{ 0xff }, 1, .short, .none }, - .{ .dec, .m, .{ .rm32 }, .{ 0xff }, 1, .none, .none }, - .{ .dec, .m, .{ .rm64 }, .{ 0xff }, 1, .long, .none }, - - .{ .div, .m, .{ .rm8 }, .{ 0xf6 }, 6, .none, .none }, - .{ .div, .m, .{ .rm8 }, .{ 0xf6 }, 6, .rex, .none }, - .{ .div, .m, .{ .rm16 }, .{ 0xf7 }, 6, .short, .none }, - .{ .div, .m, .{ .rm32 }, .{ 0xf7 }, 6, .none, .none }, - .{ .div, .m, .{ .rm64 }, .{ 0xf7 }, 6, .long, .none }, - - .{ .endbr32, .z, .{}, .{ 0xf3, 0x0f, 0x1e, 0xfb }, 0, .none, .none }, - - .{ .endbr64, .z, .{}, .{ 0xf3, 0x0f, 0x1e, 0xfa }, 0, .none, .none }, - - .{ .enqcmd, .rm, .{ .r32, .m }, .{ 0xf2, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, - .{ .enqcmd, .rm, .{ .r64, .m }, .{ 0xf2, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, - - .{ .enqcmds, .rm, .{ .r32, .m }, .{ 0xf3, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, - .{ .enqcmds, .rm, .{ .r64, .m }, .{ 0xf3, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, - - .{ .enter, .ii, .{ .imm16, .imm8 }, .{ 0xc8 }, 0, .none, .none }, - - .{ .hlt, .z, .{}, .{ 0xf4 }, 0, .none, .none }, - - .{ .hreset, .ia, .{ .imm8 }, .{ 0xf3, 0x0f, 0x3a, 0xf0 }, 0, .none, .hreset }, - .{ .hreset, .ia, .{ .imm8, .eax }, .{ 0xf3, 0x0f, 0x3a, 0xf0 }, 0, .none, .hreset }, - - .{ .idiv, .m, .{ .rm8 }, .{ 0xf6 }, 7, .none, .none }, - .{ .idiv, .m, .{ .rm8 }, .{ 0xf6 }, 7, .rex, .none }, - .{ .idiv, .m, .{ .rm16 }, .{ 0xf7 }, 7, .short, .none }, - .{ .idiv, .m, .{ .rm32 }, .{ 0xf7 }, 7, .none, .none }, - .{ .idiv, .m, .{ .rm64 }, .{ 0xf7 }, 7, .long, .none }, - - .{ .imul, .m, .{ .rm8 }, .{ 0xf6 }, 5, .none, .none }, - .{ .imul, .m, .{ .rm8 }, .{ 0xf6 }, 5, .rex, .none }, - .{ .imul, .m, .{ .rm16, }, .{ 0xf7 }, 5, .short, .none }, - .{ .imul, .m, .{ .rm32, }, .{ 0xf7 }, 5, .none, .none }, - .{ .imul, .m, .{ .rm64, }, .{ 0xf7 }, 5, .long, .none }, - .{ .imul, .rm, .{ .r16, .rm16, }, .{ 0x0f, 0xaf }, 0, .short, .none }, - .{ .imul, .rm, .{ .r32, .rm32, }, .{ 0x0f, 0xaf }, 0, .none, .none }, - .{ .imul, .rm, .{ .r64, .rm64, }, .{ 0x0f, 0xaf }, 0, .long, .none }, - .{ .imul, .rmi, .{ .r16, .rm16, .imm8s }, .{ 0x6b }, 0, .short, .none }, - .{ .imul, .rmi, .{ .r32, .rm32, .imm8s }, .{ 0x6b }, 0, .none, .none }, - .{ .imul, .rmi, .{ .r64, .rm64, .imm8s }, .{ 0x6b }, 0, .long, .none }, - .{ .imul, .rmi, .{ .r16, .rm16, .imm16 }, .{ 0x69 }, 0, .short, .none }, - .{ .imul, .rmi, .{ .r32, .rm32, .imm32 }, .{ 0x69 }, 0, .none, .none }, - .{ .imul, .rmi, .{ .r64, .rm64, .imm32 }, .{ 0x69 }, 0, .long, .none }, - - .{ .in, .zi, .{ .al, .imm8 }, .{ 0xe4 }, 0, .none, .none }, - .{ .in, .zi, .{ .ax, .imm8 }, .{ 0xe5 }, 0, .short, .none }, - .{ .in, .zi, .{ .eax, .imm8 }, .{ 0xe5 }, 0, .none, .none }, - .{ .in, .z, .{ .al, .dx }, .{ 0xec }, 0, .none, .none }, - .{ .in, .z, .{ .ax, .dx }, .{ 0xed }, 0, .short, .none }, - .{ .in, .z, .{ .eax, .dx }, .{ 0xed }, 0, .none, .none }, - - .{ .inc, .m, .{ .rm8 }, .{ 0xfe }, 0, .none, .none }, - .{ .inc, .m, .{ .rm8 }, .{ 0xfe }, 0, .rex, .none }, - .{ .inc, .m, .{ .rm16 }, .{ 0xff }, 0, .short, .none }, - .{ .inc, .m, .{ .rm32 }, .{ 0xff }, 0, .none, .none }, - .{ .inc, .m, .{ .rm64 }, .{ 0xff }, 0, .long, .none }, - - .{ .incsspd, .m, .{ .r32 }, .{ 0xf3, 0x0f, 0xae }, 5, .none, .shstk }, - .{ .incsspq, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xae }, 5, .long, .shstk }, - - .{ .ins, .z, .{ .m8, .dx }, .{ 0x6c }, 0, .none, .none }, - .{ .ins, .z, .{ .m16, .dx }, .{ 0x6d }, 0, .short, .none }, - .{ .ins, .z, .{ .m32, .dx }, .{ 0x6d }, 0, .none, .none }, - .{ .insb, .z, .{ }, .{ 0x6c }, 0, .none, .none }, - .{ .insw, .z, .{ }, .{ 0x6d }, 0, .short, .none }, - .{ .insd, .z, .{ }, .{ 0x6d }, 0, .none, .none }, - - .{ .int3, .z, .{ }, .{ 0xcc }, 0, .none, .none }, - .{ .int, .i, .{ .imm8 }, .{ 0xcd }, 0, .none, .none }, - .{ .into, .z, .{ }, .{ 0xce }, 0, .none, .@"32bit" }, - .{ .int1, .z, .{ }, .{ 0xf1 }, 0, .none, .none }, - - .{ .invd, .z, .{}, .{ 0x0f, 0x08 }, 0, .none, .none }, - - .{ .invlpg, .m, .{ .m }, .{ 0x0f, 0x01 }, 7, .none, .none }, - - .{ .invpcid, .rm, .{ .r32, .m128 }, .{ 0x66, 0x0f, 0x38, 0x82 }, 0, .none, .@"invpcid 32bit" }, - .{ .invpcid, .rm, .{ .r64, .m128 }, .{ 0x66, 0x0f, 0x38, 0x82 }, 0, .none, .@"invpcid 64bit" }, - - .{ .iretw, .z, .{}, .{ 0xcf }, 0, .short, .none }, - .{ .iretd, .z, .{}, .{ 0xcf }, 0, .none, .none }, - .{ .iret, .z, .{}, .{ 0xcf }, 0, .none, .none }, - .{ .iretq, .z, .{}, .{ 0xcf }, 0, .long, .none }, - - .{ .ja, .d, .{ .rel32 }, .{ 0x0f, 0x87 }, 0, .none, .none }, - .{ .jae, .d, .{ .rel32 }, .{ 0x0f, 0x83 }, 0, .none, .none }, - .{ .jb, .d, .{ .rel32 }, .{ 0x0f, 0x82 }, 0, .none, .none }, - .{ .jbe, .d, .{ .rel32 }, .{ 0x0f, 0x86 }, 0, .none, .none }, - .{ .jc, .d, .{ .rel32 }, .{ 0x0f, 0x82 }, 0, .none, .none }, - .{ .jcxz, .d, .{ .rel8 }, .{ 0xe3 }, 0, .short, .@"32bit" }, - .{ .jecxz, .d, .{ .rel8 }, .{ 0xe3 }, 0, .none, .@"32bit" }, - .{ .jrcxz, .d, .{ .rel8 }, .{ 0xe3 }, 0, .none, .@"64bit" }, - .{ .je, .d, .{ .rel32 }, .{ 0x0f, 0x84 }, 0, .none, .none }, - .{ .jg, .d, .{ .rel32 }, .{ 0x0f, 0x8f }, 0, .none, .none }, - .{ .jge, .d, .{ .rel32 }, .{ 0x0f, 0x8d }, 0, .none, .none }, - .{ .jl, .d, .{ .rel32 }, .{ 0x0f, 0x8c }, 0, .none, .none }, - .{ .jle, .d, .{ .rel32 }, .{ 0x0f, 0x8e }, 0, .none, .none }, - .{ .jna, .d, .{ .rel32 }, .{ 0x0f, 0x86 }, 0, .none, .none }, - .{ .jnae, .d, .{ .rel32 }, .{ 0x0f, 0x82 }, 0, .none, .none }, - .{ .jnb, .d, .{ .rel32 }, .{ 0x0f, 0x83 }, 0, .none, .none }, - .{ .jnbe, .d, .{ .rel32 }, .{ 0x0f, 0x87 }, 0, .none, .none }, - .{ .jnc, .d, .{ .rel32 }, .{ 0x0f, 0x83 }, 0, .none, .none }, - .{ .jne, .d, .{ .rel32 }, .{ 0x0f, 0x85 }, 0, .none, .none }, - .{ .jng, .d, .{ .rel32 }, .{ 0x0f, 0x8e }, 0, .none, .none }, - .{ .jnge, .d, .{ .rel32 }, .{ 0x0f, 0x8c }, 0, .none, .none }, - .{ .jnl, .d, .{ .rel32 }, .{ 0x0f, 0x8d }, 0, .none, .none }, - .{ .jnle, .d, .{ .rel32 }, .{ 0x0f, 0x8f }, 0, .none, .none }, - .{ .jno, .d, .{ .rel32 }, .{ 0x0f, 0x81 }, 0, .none, .none }, - .{ .jnp, .d, .{ .rel32 }, .{ 0x0f, 0x8b }, 0, .none, .none }, - .{ .jns, .d, .{ .rel32 }, .{ 0x0f, 0x89 }, 0, .none, .none }, - .{ .jnz, .d, .{ .rel32 }, .{ 0x0f, 0x85 }, 0, .none, .none }, - .{ .jo, .d, .{ .rel32 }, .{ 0x0f, 0x80 }, 0, .none, .none }, - .{ .jp, .d, .{ .rel32 }, .{ 0x0f, 0x8a }, 0, .none, .none }, - .{ .jpe, .d, .{ .rel32 }, .{ 0x0f, 0x8a }, 0, .none, .none }, - .{ .jpo, .d, .{ .rel32 }, .{ 0x0f, 0x8b }, 0, .none, .none }, - .{ .js, .d, .{ .rel32 }, .{ 0x0f, 0x88 }, 0, .none, .none }, - .{ .jz, .d, .{ .rel32 }, .{ 0x0f, 0x84 }, 0, .none, .none }, - - .{ .jmp, .d, .{ .rel32 }, .{ 0xe9 }, 0, .none, .none }, - .{ .jmp, .m, .{ .rm64 }, .{ 0xff }, 4, .none, .none }, - - .{ .lahf, .z, .{}, .{ 0x9f }, 0, .none, .@"32bit" }, - .{ .lahf, .z, .{}, .{ 0x9f }, 0, .none, .sahf }, - - .{ .lar, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x02 }, 0, .none, .none }, - .{ .lar, .rm, .{ .r32, .r32_m16 }, .{ 0x0f, 0x02 }, 0, .none, .none }, - - .{ .lea, .rm, .{ .r16, .m }, .{ 0x8d }, 0, .short, .none }, - .{ .lea, .rm, .{ .r32, .m }, .{ 0x8d }, 0, .none, .none }, - .{ .lea, .rm, .{ .r64, .m }, .{ 0x8d }, 0, .long, .none }, - - .{ .leave, .z, .{}, .{ 0xc9 }, 0, .none, .none }, - - .{ .lfence, .z, .{}, .{ 0x0f, 0xae, 0xe8 }, 0, .none, .none }, - - .{ .lgdt, .m, .{ .m }, .{ 0x0f, 0x01 }, 2, .none, .none }, - .{ .lidt, .m, .{ .m }, .{ 0x0f, 0x01 }, 3, .none, .none }, - - .{ .lldt, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 2, .none, .none }, - - .{ .lmsw, .m, .{ .rm16 }, .{ 0x0f, 0x01 }, 6, .none, .none }, - - .{ .lods, .z, .{ .m8 }, .{ 0xac }, 0, .none, .none }, - .{ .lods, .z, .{ .m16 }, .{ 0xad }, 0, .short, .none }, - .{ .lods, .z, .{ .m32 }, .{ 0xad }, 0, .none, .none }, - .{ .lods, .z, .{ .m64 }, .{ 0xad }, 0, .long, .none }, - .{ .lodsb, .z, .{ }, .{ 0xac }, 0, .none, .none }, - .{ .lodsw, .z, .{ }, .{ 0xad }, 0, .short, .none }, - .{ .lodsd, .z, .{ }, .{ 0xad }, 0, .none, .none }, - .{ .lodsq, .z, .{ }, .{ 0xad }, 0, .long, .none }, - - .{ .loop, .d, .{ .rel8 }, .{ 0xe2 }, 0, .none, .none }, - .{ .loope, .d, .{ .rel8 }, .{ 0xe1 }, 0, .none, .none }, - .{ .loopne, .d, .{ .rel8 }, .{ 0xe0 }, 0, .none, .none }, - - .{ .lsl, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x03 }, 0, .none, .none }, - .{ .lsl, .rm, .{ .r32, .r32_m16 }, .{ 0x0f, 0x03 }, 0, .none, .none }, - .{ .lsl, .rm, .{ .r64, .r32_m16 }, .{ 0x0f, 0x03 }, 0, .none, .none }, - - .{ .ltr, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 3, .none, .none }, - - .{ .mfence, .z, .{}, .{ 0x0f, 0xae, 0xf0 }, 0, .none, .none }, - - .{ .mov, .mr, .{ .rm8, .r8 }, .{ 0x88 }, 0, .none, .none }, - .{ .mov, .mr, .{ .rm8, .r8 }, .{ 0x88 }, 0, .rex, .none }, - .{ .mov, .mr, .{ .rm16, .r16 }, .{ 0x89 }, 0, .short, .none }, - .{ .mov, .mr, .{ .rm32, .r32 }, .{ 0x89 }, 0, .none, .none }, - .{ .mov, .mr, .{ .rm64, .r64 }, .{ 0x89 }, 0, .long, .none }, - .{ .mov, .rm, .{ .r8, .rm8 }, .{ 0x8a }, 0, .none, .none }, - .{ .mov, .rm, .{ .r8, .rm8 }, .{ 0x8a }, 0, .rex, .none }, - .{ .mov, .rm, .{ .r16, .rm16 }, .{ 0x8b }, 0, .short, .none }, - .{ .mov, .rm, .{ .r32, .rm32 }, .{ 0x8b }, 0, .none, .none }, - .{ .mov, .rm, .{ .r64, .rm64 }, .{ 0x8b }, 0, .long, .none }, - .{ .mov, .mr, .{ .rm16, .sreg }, .{ 0x8c }, 0, .short, .none }, - .{ .mov, .mr, .{ .r32_m16, .sreg }, .{ 0x8c }, 0, .none, .none }, - .{ .mov, .mr, .{ .r64_m16, .sreg }, .{ 0x8c }, 0, .long, .none }, - .{ .mov, .rm, .{ .sreg, .rm16 }, .{ 0x8e }, 0, .short, .none }, - .{ .mov, .rm, .{ .sreg, .r32_m16 }, .{ 0x8e }, 0, .none, .none }, - .{ .mov, .rm, .{ .sreg, .r64_m16 }, .{ 0x8e }, 0, .long, .none }, - .{ .mov, .fd, .{ .al, .moffs }, .{ 0xa0 }, 0, .none, .none }, - .{ .mov, .fd, .{ .ax, .moffs }, .{ 0xa1 }, 0, .short, .none }, - .{ .mov, .fd, .{ .eax, .moffs }, .{ 0xa1 }, 0, .none, .none }, - .{ .mov, .fd, .{ .rax, .moffs }, .{ 0xa1 }, 0, .long, .none }, - .{ .mov, .td, .{ .moffs, .al }, .{ 0xa2 }, 0, .none, .none }, - .{ .mov, .td, .{ .moffs, .ax }, .{ 0xa3 }, 0, .short, .none }, - .{ .mov, .td, .{ .moffs, .eax }, .{ 0xa3 }, 0, .none, .none }, - .{ .mov, .td, .{ .moffs, .rax }, .{ 0xa3 }, 0, .long, .none }, - .{ .mov, .oi, .{ .r8, .imm8 }, .{ 0xb0 }, 0, .none, .none }, - .{ .mov, .oi, .{ .r8, .imm8 }, .{ 0xb0 }, 0, .rex, .none }, - .{ .mov, .oi, .{ .r16, .imm16 }, .{ 0xb8 }, 0, .short, .none }, - .{ .mov, .oi, .{ .r32, .imm32 }, .{ 0xb8 }, 0, .none, .none }, - .{ .mov, .oi, .{ .r64, .imm64 }, .{ 0xb8 }, 0, .long, .none }, - .{ .mov, .mi, .{ .rm8, .imm8 }, .{ 0xc6 }, 0, .none, .none }, - .{ .mov, .mi, .{ .rm8, .imm8 }, .{ 0xc6 }, 0, .rex, .none }, - .{ .mov, .mi, .{ .rm16, .imm16 }, .{ 0xc7 }, 0, .short, .none }, - .{ .mov, .mi, .{ .rm32, .imm32 }, .{ 0xc7 }, 0, .none, .none }, - .{ .mov, .mi, .{ .rm64, .imm32s }, .{ 0xc7 }, 0, .long, .none }, - - .{ .mov, .mr, .{ .r32, .cr }, .{ 0x0f, 0x20 }, 0, .none, .@"32bit" }, - .{ .mov, .mr, .{ .r64, .cr }, .{ 0x0f, 0x20 }, 0, .none, .@"64bit" }, - .{ .mov, .rm, .{ .cr, .r32 }, .{ 0x0f, 0x22 }, 0, .none, .@"32bit" }, - .{ .mov, .rm, .{ .cr, .r64 }, .{ 0x0f, 0x22 }, 0, .none, .@"64bit" }, - - .{ .mov, .mr, .{ .r32, .dr }, .{ 0x0f, 0x21 }, 0, .none, .@"32bit" }, - .{ .mov, .mr, .{ .r64, .dr }, .{ 0x0f, 0x21 }, 0, .none, .@"64bit" }, - .{ .mov, .rm, .{ .dr, .r32 }, .{ 0x0f, 0x23 }, 0, .none, .@"32bit" }, - .{ .mov, .rm, .{ .dr, .r64 }, .{ 0x0f, 0x23 }, 0, .none, .@"64bit" }, - - .{ .movbe, .rm, .{ .r16, .m16 }, .{ 0x0f, 0x38, 0xf0 }, 0, .short, .movbe }, - .{ .movbe, .rm, .{ .r32, .m32 }, .{ 0x0f, 0x38, 0xf0 }, 0, .none, .movbe }, - .{ .movbe, .rm, .{ .r64, .m64 }, .{ 0x0f, 0x38, 0xf0 }, 0, .long, .movbe }, - .{ .movbe, .mr, .{ .m16, .r16 }, .{ 0x0f, 0x38, 0xf1 }, 0, .short, .movbe }, - .{ .movbe, .mr, .{ .m32, .r32 }, .{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe }, - .{ .movbe, .mr, .{ .m64, .r64 }, .{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe }, - - .{ .movs, .z, .{ .m8, .m8 }, .{ 0xa4 }, 0, .none, .none }, - .{ .movs, .z, .{ .m16, .m16 }, .{ 0xa5 }, 0, .short, .none }, - .{ .movs, .z, .{ .m32, .m32 }, .{ 0xa5 }, 0, .none, .none }, - .{ .movs, .z, .{ .m64, .m64 }, .{ 0xa5 }, 0, .long, .none }, - .{ .movsb, .z, .{ }, .{ 0xa4 }, 0, .none, .none }, - .{ .movsw, .z, .{ }, .{ 0xa5 }, 0, .short, .none }, - .{ .movsd, .z, .{ }, .{ 0xa5 }, 0, .none, .none }, - .{ .movsq, .z, .{ }, .{ 0xa5 }, 0, .long, .none }, - - .{ .movsx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xbe }, 0, .short, .none }, - .{ .movsx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xbe }, 0, .rex_short, .none }, - .{ .movsx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xbe }, 0, .none, .none }, - .{ .movsx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xbe }, 0, .rex, .none }, - .{ .movsx, .rm, .{ .r64, .rm8 }, .{ 0x0f, 0xbe }, 0, .long, .none }, - .{ .movsx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xbf }, 0, .none, .none }, - .{ .movsx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xbf }, 0, .rex, .none }, - .{ .movsx, .rm, .{ .r64, .rm16 }, .{ 0x0f, 0xbf }, 0, .long, .none }, - - // This instruction is discouraged. - .{ .movsxd, .rm, .{ .r32, .rm32 }, .{ 0x63 }, 0, .none, .@"64bit" }, - .{ .movsxd, .rm, .{ .r64, .rm32 }, .{ 0x63 }, 0, .long, .@"64bit" }, - - .{ .movzx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xb6 }, 0, .short, .none }, - .{ .movzx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xb6 }, 0, .rex_short, .none }, - .{ .movzx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xb6 }, 0, .none, .none }, - .{ .movzx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xb6 }, 0, .rex, .none }, - .{ .movzx, .rm, .{ .r64, .rm8 }, .{ 0x0f, 0xb6 }, 0, .long, .none }, - .{ .movzx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xb7 }, 0, .none, .none }, - .{ .movzx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xb7 }, 0, .rex, .none }, - .{ .movzx, .rm, .{ .r64, .rm16 }, .{ 0x0f, 0xb7 }, 0, .long, .none }, - - .{ .mul, .m, .{ .rm8 }, .{ 0xf6 }, 4, .none, .none }, - .{ .mul, .m, .{ .rm8 }, .{ 0xf6 }, 4, .rex, .none }, - .{ .mul, .m, .{ .rm16 }, .{ 0xf7 }, 4, .short, .none }, - .{ .mul, .m, .{ .rm32 }, .{ 0xf7 }, 4, .none, .none }, - .{ .mul, .m, .{ .rm64 }, .{ 0xf7 }, 4, .long, .none }, - - .{ .neg, .m, .{ .rm8 }, .{ 0xf6 }, 3, .none, .none }, - .{ .neg, .m, .{ .rm8 }, .{ 0xf6 }, 3, .rex, .none }, - .{ .neg, .m, .{ .rm16 }, .{ 0xf7 }, 3, .short, .none }, - .{ .neg, .m, .{ .rm32 }, .{ 0xf7 }, 3, .none, .none }, - .{ .neg, .m, .{ .rm64 }, .{ 0xf7 }, 3, .long, .none }, - - .{ .nop, .z, .{}, .{ 0x90 }, 0, .none, .none }, - - .{ .not, .m, .{ .rm8 }, .{ 0xf6 }, 2, .none, .none }, - .{ .not, .m, .{ .rm8 }, .{ 0xf6 }, 2, .rex, .none }, - .{ .not, .m, .{ .rm16 }, .{ 0xf7 }, 2, .short, .none }, - .{ .not, .m, .{ .rm32 }, .{ 0xf7 }, 2, .none, .none }, - .{ .not, .m, .{ .rm64 }, .{ 0xf7 }, 2, .long, .none }, - - .{ .@"or", .zi, .{ .al, .imm8 }, .{ 0x0c }, 0, .none, .none }, - .{ .@"or", .zi, .{ .ax, .imm16 }, .{ 0x0d }, 0, .short, .none }, - .{ .@"or", .zi, .{ .eax, .imm32 }, .{ 0x0d }, 0, .none, .none }, - .{ .@"or", .zi, .{ .rax, .imm32s }, .{ 0x0d }, 0, .long, .none }, - .{ .@"or", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 1, .none, .none }, - .{ .@"or", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 1, .rex, .none }, - .{ .@"or", .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 1, .short, .none }, - .{ .@"or", .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 1, .none, .none }, - .{ .@"or", .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 1, .long, .none }, - .{ .@"or", .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 1, .short, .none }, - .{ .@"or", .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 1, .none, .none }, - .{ .@"or", .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 1, .long, .none }, - .{ .@"or", .mr, .{ .rm8, .r8 }, .{ 0x08 }, 0, .none, .none }, - .{ .@"or", .mr, .{ .rm8, .r8 }, .{ 0x08 }, 0, .rex, .none }, - .{ .@"or", .mr, .{ .rm16, .r16 }, .{ 0x09 }, 0, .short, .none }, - .{ .@"or", .mr, .{ .rm32, .r32 }, .{ 0x09 }, 0, .none, .none }, - .{ .@"or", .mr, .{ .rm64, .r64 }, .{ 0x09 }, 0, .long, .none }, - .{ .@"or", .rm, .{ .r8, .rm8 }, .{ 0x0a }, 0, .none, .none }, - .{ .@"or", .rm, .{ .r8, .rm8 }, .{ 0x0a }, 0, .rex, .none }, - .{ .@"or", .rm, .{ .r16, .rm16 }, .{ 0x0b }, 0, .short, .none }, - .{ .@"or", .rm, .{ .r32, .rm32 }, .{ 0x0b }, 0, .none, .none }, - .{ .@"or", .rm, .{ .r64, .rm64 }, .{ 0x0b }, 0, .long, .none }, - - .{ .out, .zi, .{ .imm8, .al }, .{ 0xe6 }, 0, .none, .none }, - .{ .out, .zi, .{ .imm8, .ax }, .{ 0xe7 }, 0, .short, .none }, - .{ .out, .zi, .{ .imm8, .eax }, .{ 0xe7 }, 0, .none, .none }, - .{ .out, .z, .{ .dx, .al }, .{ 0xee }, 0, .none, .none }, - .{ .out, .z, .{ .dx, .ax }, .{ 0xef }, 0, .short, .none }, - .{ .out, .z, .{ .dx, .eax }, .{ 0xef }, 0, .none, .none }, - - .{ .outs, .z, .{ .dx, .m8 }, .{ 0x6e }, 0, .none, .none }, - .{ .outs, .z, .{ .dx, .m16 }, .{ 0x6f }, 0, .short, .none }, - .{ .outs, .z, .{ .dx, .m32 }, .{ 0x6f }, 0, .none, .none }, - .{ .outsb, .z, .{ }, .{ 0x6e }, 0, .none, .none }, - .{ .outsw, .z, .{ }, .{ 0x6f }, 0, .short, .none }, - .{ .outsd, .z, .{ }, .{ 0x6f }, 0, .none, .none }, - - .{ .pause, .z, .{}, .{ 0xf3, 0x90 }, 0, .none, .none }, - - .{ .pop, .o, .{ .r16 }, .{ 0x58 }, 0, .short, .none }, - .{ .pop, .o, .{ .r64 }, .{ 0x58 }, 0, .none, .none }, - .{ .pop, .m, .{ .rm16 }, .{ 0x8f }, 0, .short, .none }, - .{ .pop, .m, .{ .rm64 }, .{ 0x8f }, 0, .none, .none }, - - .{ .popf, .z, .{}, .{ 0x9d }, 0, .short, .none }, - .{ .popfd, .z, .{}, .{ 0x9d }, 0, .none, .@"32bit" }, - .{ .popfq, .z, .{}, .{ 0x9d }, 0, .none, .@"64bit" }, - - .{ .push, .o, .{ .r16 }, .{ 0x50 }, 0, .short, .none }, - .{ .push, .o, .{ .r64 }, .{ 0x50 }, 0, .none, .none }, - .{ .push, .m, .{ .rm16 }, .{ 0xff }, 6, .short, .none }, - .{ .push, .m, .{ .rm64 }, .{ 0xff }, 6, .none, .none }, - .{ .push, .i, .{ .imm8 }, .{ 0x6a }, 0, .none, .none }, - .{ .push, .i, .{ .imm16 }, .{ 0x68 }, 0, .short, .none }, - .{ .push, .i, .{ .imm32 }, .{ 0x68 }, 0, .none, .none }, - - .{ .pushfq, .z, .{}, .{ 0x9c }, 0, .none, .none }, - - .{ .ret, .z, .{}, .{ 0xc3 }, 0, .none, .none }, - - .{ .rcl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 2, .none, .none }, - .{ .rcl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 2, .rex, .none }, - .{ .rcl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 2, .none, .none }, - .{ .rcl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 2, .rex, .none }, - .{ .rcl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 2, .none, .none }, - .{ .rcl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 2, .rex, .none }, - .{ .rcl, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 2, .short, .none }, - .{ .rcl, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 2, .short, .none }, - .{ .rcl, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 2, .short, .none }, - .{ .rcl, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 2, .none, .none }, - .{ .rcl, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 2, .long, .none }, - .{ .rcl, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 2, .none, .none }, - .{ .rcl, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 2, .long, .none }, - .{ .rcl, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 2, .none, .none }, - .{ .rcl, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 2, .long, .none }, - - .{ .rcr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 3, .none, .none }, - .{ .rcr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 3, .rex, .none }, - .{ .rcr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 3, .none, .none }, - .{ .rcr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 3, .rex, .none }, - .{ .rcr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 3, .none, .none }, - .{ .rcr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 3, .rex, .none }, - .{ .rcr, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 3, .short, .none }, - .{ .rcr, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 3, .short, .none }, - .{ .rcr, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 3, .short, .none }, - .{ .rcr, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 3, .none, .none }, - .{ .rcr, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 3, .long, .none }, - .{ .rcr, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 3, .none, .none }, - .{ .rcr, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 3, .long, .none }, - .{ .rcr, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 3, .none, .none }, - .{ .rcr, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 3, .long, .none }, - - .{ .rdfsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 0, .none, .fsgsbase }, - .{ .rdfsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 0, .long, .fsgsbase }, - .{ .rdgsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 1, .none, .fsgsbase }, - .{ .rdgsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 1, .long, .fsgsbase }, - - .{ .rdmsr, .z, .{}, .{ 0x0f, 0x32 }, 0, .none, .none }, - - .{ .rdpid, .m, .{ .r32 }, .{ 0xf3, 0x0f, 0xc7 }, 7, .none, .@"rdpid 32bit" }, - .{ .rdpid, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xc7 }, 7, .none, .@"rdpid 64bit" }, - - .{ .rdpkru, .z, .{}, .{ 0x0f, 0x01, 0xee }, 0, .none, .pku }, - - .{ .rdpmc, .z, .{}, .{ 0x0f, 0x33 }, 0, .none, .none }, - - .{ .rdrand, .m, .{ .r16 }, .{ 0x0f, 0xc7 }, 6, .short, .rdrnd }, - .{ .rdrand, .m, .{ .r32 }, .{ 0x0f, 0xc7 }, 6, .none, .rdrnd }, - .{ .rdrand, .m, .{ .r64 }, .{ 0x0f, 0xc7 }, 6, .long, .rdrnd }, - - .{ .rdseed, .m, .{ .r16 }, .{ 0x0f, 0xc7 }, 7, .short, .rdseed }, - .{ .rdseed, .m, .{ .r32 }, .{ 0x0f, 0xc7 }, 7, .none, .rdseed }, - .{ .rdseed, .m, .{ .r64 }, .{ 0x0f, 0xc7 }, 7, .long, .rdseed }, - - .{ .rdsspd, .m, .{ .r32 }, .{ 0xf3, 0x0f, 0x1e }, 1, .none, .shstk }, - .{ .rdsspq, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0x1e }, 1, .long, .shstk }, - - .{ .rdtsc, .z, .{}, .{ 0x0f, 0x31 }, 0, .none, .none }, - - .{ .rdtscp, .z, .{}, .{ 0x0f, 0x01, 0xf9 }, 0, .none, .none }, - - .{ .rol, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 0, .none, .none }, - .{ .rol, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 0, .rex, .none }, - .{ .rol, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 0, .none, .none }, - .{ .rol, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 0, .rex, .none }, - .{ .rol, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 0, .none, .none }, - .{ .rol, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 0, .rex, .none }, - .{ .rol, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 0, .short, .none }, - .{ .rol, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 0, .short, .none }, - .{ .rol, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 0, .short, .none }, - .{ .rol, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 0, .none, .none }, - .{ .rol, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 0, .long, .none }, - .{ .rol, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 0, .none, .none }, - .{ .rol, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 0, .long, .none }, - .{ .rol, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 0, .none, .none }, - .{ .rol, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 0, .long, .none }, - - .{ .ror, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 1, .none, .none }, - .{ .ror, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 1, .rex, .none }, - .{ .ror, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 1, .none, .none }, - .{ .ror, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 1, .rex, .none }, - .{ .ror, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 1, .none, .none }, - .{ .ror, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 1, .rex, .none }, - .{ .ror, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 1, .short, .none }, - .{ .ror, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 1, .short, .none }, - .{ .ror, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 1, .short, .none }, - .{ .ror, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 1, .none, .none }, - .{ .ror, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 1, .long, .none }, - .{ .ror, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 1, .none, .none }, - .{ .ror, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 1, .long, .none }, - .{ .ror, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 1, .none, .none }, - .{ .ror, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 1, .long, .none }, - - .{ .rsm, .z, .{}, .{ 0x0f, 0xaa }, 0, .none, .none }, - - .{ .sahf, .z, .{}, .{ 0x9e }, 0, .none, .@"32bit" }, - .{ .sahf, .z, .{}, .{ 0x9e }, 0, .none, .sahf }, - - .{ .sal, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .none, .none }, - .{ .sal, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .rex, .none }, - .{ .sal, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 4, .short, .none }, - .{ .sal, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 4, .none, .none }, - .{ .sal, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 4, .long, .none }, - .{ .sal, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .none, .none }, - .{ .sal, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .rex, .none }, - .{ .sal, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 4, .short, .none }, - .{ .sal, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 4, .none, .none }, - .{ .sal, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 4, .long, .none }, - .{ .sal, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .none, .none }, - .{ .sal, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .rex, .none }, - .{ .sal, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 4, .short, .none }, - .{ .sal, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 4, .none, .none }, - .{ .sal, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 4, .long, .none }, - - .{ .sar, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 7, .none, .none }, - .{ .sar, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 7, .rex, .none }, - .{ .sar, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 7, .short, .none }, - .{ .sar, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 7, .none, .none }, - .{ .sar, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 7, .long, .none }, - .{ .sar, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 7, .none, .none }, - .{ .sar, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 7, .rex, .none }, - .{ .sar, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 7, .short, .none }, - .{ .sar, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 7, .none, .none }, - .{ .sar, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 7, .long, .none }, - .{ .sar, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 7, .none, .none }, - .{ .sar, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 7, .rex, .none }, - .{ .sar, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 7, .short, .none }, - .{ .sar, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 7, .none, .none }, - .{ .sar, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 7, .long, .none }, - - .{ .sbb, .zi, .{ .al, .imm8 }, .{ 0x1c }, 0, .none, .none }, - .{ .sbb, .zi, .{ .ax, .imm16 }, .{ 0x1d }, 0, .short, .none }, - .{ .sbb, .zi, .{ .eax, .imm32 }, .{ 0x1d }, 0, .none, .none }, - .{ .sbb, .zi, .{ .rax, .imm32s }, .{ 0x1d }, 0, .long, .none }, - .{ .sbb, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 3, .none, .none }, - .{ .sbb, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 3, .rex, .none }, - .{ .sbb, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 3, .short, .none }, - .{ .sbb, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 3, .none, .none }, - .{ .sbb, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 3, .long, .none }, - .{ .sbb, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 3, .short, .none }, - .{ .sbb, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 3, .none, .none }, - .{ .sbb, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 3, .long, .none }, - .{ .sbb, .mr, .{ .rm8, .r8 }, .{ 0x18 }, 0, .none, .none }, - .{ .sbb, .mr, .{ .rm8, .r8 }, .{ 0x18 }, 0, .rex, .none }, - .{ .sbb, .mr, .{ .rm16, .r16 }, .{ 0x19 }, 0, .short, .none }, - .{ .sbb, .mr, .{ .rm32, .r32 }, .{ 0x19 }, 0, .none, .none }, - .{ .sbb, .mr, .{ .rm64, .r64 }, .{ 0x19 }, 0, .long, .none }, - .{ .sbb, .rm, .{ .r8, .rm8 }, .{ 0x1a }, 0, .none, .none }, - .{ .sbb, .rm, .{ .r8, .rm8 }, .{ 0x1a }, 0, .rex, .none }, - .{ .sbb, .rm, .{ .r16, .rm16 }, .{ 0x1b }, 0, .short, .none }, - .{ .sbb, .rm, .{ .r32, .rm32 }, .{ 0x1b }, 0, .none, .none }, - .{ .sbb, .rm, .{ .r64, .rm64 }, .{ 0x1b }, 0, .long, .none }, - - .{ .scas, .z, .{ .m8 }, .{ 0xae }, 0, .none, .none }, - .{ .scas, .z, .{ .m16 }, .{ 0xaf }, 0, .short, .none }, - .{ .scas, .z, .{ .m32 }, .{ 0xaf }, 0, .none, .none }, - .{ .scas, .z, .{ .m64 }, .{ 0xaf }, 0, .long, .none }, - .{ .scasb, .z, .{ }, .{ 0xae }, 0, .none, .none }, - .{ .scasw, .z, .{ }, .{ 0xaf }, 0, .short, .none }, - .{ .scasd, .z, .{ }, .{ 0xaf }, 0, .none, .none }, - .{ .scasq, .z, .{ }, .{ 0xaf }, 0, .long, .none }, - - .{ .senduipi, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xc7 }, 6, .none, .uintr }, - - .{ .serialize, .z, .{}, .{ 0x0f, 0x01, 0xe8 }, 0, .none, .serialize }, - - .{ .seta, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .none, .none }, - .{ .seta, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .rex, .none }, - .{ .setae, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .none, .none }, - .{ .setae, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .rex, .none }, - .{ .setb, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .none, .none }, - .{ .setb, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .rex, .none }, - .{ .setbe, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .none, .none }, - .{ .setbe, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .rex, .none }, - .{ .setc, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .none, .none }, - .{ .setc, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .rex, .none }, - .{ .sete, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .none, .none }, - .{ .sete, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .rex, .none }, - .{ .setg, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .none, .none }, - .{ .setg, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .rex, .none }, - .{ .setge, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .none, .none }, - .{ .setge, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .rex, .none }, - .{ .setl, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .none, .none }, - .{ .setl, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .rex, .none }, - .{ .setle, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .none, .none }, - .{ .setle, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .rex, .none }, - .{ .setna, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .none, .none }, - .{ .setna, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .rex, .none }, - .{ .setnae, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .none, .none }, - .{ .setnae, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .rex, .none }, - .{ .setnb, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .none, .none }, - .{ .setnb, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .rex, .none }, - .{ .setnbe, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .none, .none }, - .{ .setnbe, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .rex, .none }, - .{ .setnc, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .none, .none }, - .{ .setnc, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .rex, .none }, - .{ .setne, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .none, .none }, - .{ .setne, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .rex, .none }, - .{ .setng, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .none, .none }, - .{ .setng, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .rex, .none }, - .{ .setnge, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .none, .none }, - .{ .setnge, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .rex, .none }, - .{ .setnl, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .none, .none }, - .{ .setnl, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .rex, .none }, - .{ .setnle, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .none, .none }, - .{ .setnle, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .rex, .none }, - .{ .setno, .m, .{ .rm8 }, .{ 0x0f, 0x91 }, 0, .none, .none }, - .{ .setno, .m, .{ .rm8 }, .{ 0x0f, 0x91 }, 0, .rex, .none }, - .{ .setnp, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .none, .none }, - .{ .setnp, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .rex, .none }, - .{ .setns, .m, .{ .rm8 }, .{ 0x0f, 0x99 }, 0, .none, .none }, - .{ .setns, .m, .{ .rm8 }, .{ 0x0f, 0x99 }, 0, .rex, .none }, - .{ .setnz, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .none, .none }, - .{ .setnz, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .rex, .none }, - .{ .seto, .m, .{ .rm8 }, .{ 0x0f, 0x90 }, 0, .none, .none }, - .{ .seto, .m, .{ .rm8 }, .{ 0x0f, 0x90 }, 0, .rex, .none }, - .{ .setp, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .none, .none }, - .{ .setp, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .rex, .none }, - .{ .setpe, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .none, .none }, - .{ .setpe, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .rex, .none }, - .{ .setpo, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .none, .none }, - .{ .setpo, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .rex, .none }, - .{ .sets, .m, .{ .rm8 }, .{ 0x0f, 0x98 }, 0, .none, .none }, - .{ .sets, .m, .{ .rm8 }, .{ 0x0f, 0x98 }, 0, .rex, .none }, - .{ .setz, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .none, .none }, - .{ .setz, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .rex, .none }, - - .{ .sfence, .z, .{}, .{ 0x0f, 0xae, 0xf8 }, 0, .none, .none }, - - .{ .sidt, .m, .{ .m }, .{ 0x0f, 0x01 }, 1, .none, .none }, - - .{ .sldt, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 0, .none, .none }, - - .{ .smsw, .m, .{ .rm16 }, .{ 0x0f, 0x01 }, 4, .short, .none }, - .{ .smsw, .m, .{ .r32_m16 }, .{ 0x0f, 0x01 }, 4, .none, .none }, - .{ .smsw, .m, .{ .r64_m16 }, .{ 0x0f, 0x01 }, 4, .long, .none }, - - .{ .shl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .none, .none }, - .{ .shl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .rex, .none }, - .{ .shl, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 4, .short, .none }, - .{ .shl, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 4, .none, .none }, - .{ .shl, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 4, .long, .none }, - .{ .shl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .none, .none }, - .{ .shl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .rex, .none }, - .{ .shl, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 4, .short, .none }, - .{ .shl, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 4, .none, .none }, - .{ .shl, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 4, .long, .none }, - .{ .shl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .none, .none }, - .{ .shl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .rex, .none }, - .{ .shl, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 4, .short, .none }, - .{ .shl, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 4, .none, .none }, - .{ .shl, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 4, .long, .none }, - - .{ .shld, .mri, .{ .rm16, .r16, .imm8 }, .{ 0x0f, 0xa4 }, 0, .short, .none }, - .{ .shld, .mrc, .{ .rm16, .r16, .cl }, .{ 0x0f, 0xa5 }, 0, .short, .none }, - .{ .shld, .mri, .{ .rm32, .r32, .imm8 }, .{ 0x0f, 0xa4 }, 0, .none, .none }, - .{ .shld, .mri, .{ .rm64, .r64, .imm8 }, .{ 0x0f, 0xa4 }, 0, .long, .none }, - .{ .shld, .mrc, .{ .rm32, .r32, .cl }, .{ 0x0f, 0xa5 }, 0, .none, .none }, - .{ .shld, .mrc, .{ .rm64, .r64, .cl }, .{ 0x0f, 0xa5 }, 0, .long, .none }, - - .{ .shr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 5, .none, .none }, - .{ .shr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 5, .rex, .none }, - .{ .shr, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 5, .short, .none }, - .{ .shr, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 5, .none, .none }, - .{ .shr, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 5, .long, .none }, - .{ .shr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 5, .none, .none }, - .{ .shr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 5, .rex, .none }, - .{ .shr, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 5, .short, .none }, - .{ .shr, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 5, .none, .none }, - .{ .shr, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 5, .long, .none }, - .{ .shr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 5, .none, .none }, - .{ .shr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 5, .rex, .none }, - .{ .shr, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 5, .short, .none }, - .{ .shr, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 5, .none, .none }, - .{ .shr, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 5, .long, .none }, - - .{ .shrd, .mri, .{ .rm16, .r16, .imm8 }, .{ 0x0f, 0xac }, 0, .short, .none }, - .{ .shrd, .mrc, .{ .rm16, .r16, .cl }, .{ 0x0f, 0xad }, 0, .short, .none }, - .{ .shrd, .mri, .{ .rm32, .r32, .imm8 }, .{ 0x0f, 0xac }, 0, .none, .none }, - .{ .shrd, .mri, .{ .rm64, .r64, .imm8 }, .{ 0x0f, 0xac }, 0, .long, .none }, - .{ .shrd, .mrc, .{ .rm32, .r32, .cl }, .{ 0x0f, 0xad }, 0, .none, .none }, - .{ .shrd, .mrc, .{ .rm64, .r64, .cl }, .{ 0x0f, 0xad }, 0, .long, .none }, - - .{ .stac, .z, .{}, .{ 0x0f, 0x01, 0xcb }, 0, .none, .smap }, - - .{ .stc, .z, .{}, .{ 0xf9 }, 0, .none, .none }, - - .{ .std, .z, .{}, .{ 0xfd }, 0, .none, .none }, - - .{ .sti, .z, .{}, .{ 0xfb }, 0, .none, .none }, - - .{ .str, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 1, .none, .none }, - - .{ .stui, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr }, - - .{ .stos, .z, .{ .m8 }, .{ 0xaa }, 0, .none, .none }, - .{ .stos, .z, .{ .m16 }, .{ 0xab }, 0, .short, .none }, - .{ .stos, .z, .{ .m32 }, .{ 0xab }, 0, .none, .none }, - .{ .stos, .z, .{ .m64 }, .{ 0xab }, 0, .long, .none }, - .{ .stosb, .z, .{ }, .{ 0xaa }, 0, .none, .none }, - .{ .stosw, .z, .{ }, .{ 0xab }, 0, .short, .none }, - .{ .stosd, .z, .{ }, .{ 0xab }, 0, .none, .none }, - .{ .stosq, .z, .{ }, .{ 0xab }, 0, .long, .none }, - - .{ .sub, .zi, .{ .al, .imm8 }, .{ 0x2c }, 0, .none, .none }, - .{ .sub, .zi, .{ .ax, .imm16 }, .{ 0x2d }, 0, .short, .none }, - .{ .sub, .zi, .{ .eax, .imm32 }, .{ 0x2d }, 0, .none, .none }, - .{ .sub, .zi, .{ .rax, .imm32s }, .{ 0x2d }, 0, .long, .none }, - .{ .sub, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 5, .none, .none }, - .{ .sub, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 5, .rex, .none }, - .{ .sub, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 5, .short, .none }, - .{ .sub, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 5, .none, .none }, - .{ .sub, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 5, .long, .none }, - .{ .sub, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 5, .short, .none }, - .{ .sub, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 5, .none, .none }, - .{ .sub, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 5, .long, .none }, - .{ .sub, .mr, .{ .rm8, .r8 }, .{ 0x28 }, 0, .none, .none }, - .{ .sub, .mr, .{ .rm8, .r8 }, .{ 0x28 }, 0, .rex, .none }, - .{ .sub, .mr, .{ .rm16, .r16 }, .{ 0x29 }, 0, .short, .none }, - .{ .sub, .mr, .{ .rm32, .r32 }, .{ 0x29 }, 0, .none, .none }, - .{ .sub, .mr, .{ .rm64, .r64 }, .{ 0x29 }, 0, .long, .none }, - .{ .sub, .rm, .{ .r8, .rm8 }, .{ 0x2a }, 0, .none, .none }, - .{ .sub, .rm, .{ .r8, .rm8 }, .{ 0x2a }, 0, .rex, .none }, - .{ .sub, .rm, .{ .r16, .rm16 }, .{ 0x2b }, 0, .short, .none }, - .{ .sub, .rm, .{ .r32, .rm32 }, .{ 0x2b }, 0, .none, .none }, - .{ .sub, .rm, .{ .r64, .rm64 }, .{ 0x2b }, 0, .long, .none }, - - .{ .swapgs, .z, .{}, .{ 0x0f, 0x01, 0xf8 }, 0, .none, .@"64bit" }, - - .{ .syscall, .z, .{}, .{ 0x0f, 0x05 }, 0, .none, .@"64bit" }, - - .{ .sysenter, .z, .{}, .{ 0x0f, 0x34 }, 0, .none, .none }, - - .{ .sysexit, .z, .{}, .{ 0x0f, 0x35 }, 0, .none, .none }, - .{ .sysexit, .z, .{}, .{ 0x0f, 0x35 }, 0, .long, .none }, - - .{ .sysret, .z, .{}, .{ 0x0f, 0x37 }, 0, .none, .none }, - .{ .sysret, .z, .{}, .{ 0x0f, 0x37 }, 0, .long, .none }, - - .{ .@"test", .zi, .{ .al, .imm8 }, .{ 0xa8 }, 0, .none, .none }, - .{ .@"test", .zi, .{ .ax, .imm16 }, .{ 0xa9 }, 0, .short, .none }, - .{ .@"test", .zi, .{ .eax, .imm32 }, .{ 0xa9 }, 0, .none, .none }, - .{ .@"test", .zi, .{ .rax, .imm32s }, .{ 0xa9 }, 0, .long, .none }, - .{ .@"test", .mi, .{ .rm8, .imm8 }, .{ 0xf6 }, 0, .none, .none }, - .{ .@"test", .mi, .{ .rm8, .imm8 }, .{ 0xf6 }, 0, .rex, .none }, - .{ .@"test", .mi, .{ .rm16, .imm16 }, .{ 0xf7 }, 0, .short, .none }, - .{ .@"test", .mi, .{ .rm32, .imm32 }, .{ 0xf7 }, 0, .none, .none }, - .{ .@"test", .mi, .{ .rm64, .imm32s }, .{ 0xf7 }, 0, .long, .none }, - .{ .@"test", .mr, .{ .rm8, .r8 }, .{ 0x84 }, 0, .none, .none }, - .{ .@"test", .mr, .{ .rm8, .r8 }, .{ 0x84 }, 0, .rex, .none }, - .{ .@"test", .mr, .{ .rm16, .r16 }, .{ 0x85 }, 0, .short, .none }, - .{ .@"test", .mr, .{ .rm32, .r32 }, .{ 0x85 }, 0, .none, .none }, - .{ .@"test", .mr, .{ .rm64, .r64 }, .{ 0x85 }, 0, .long, .none }, - - .{ .testui, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xed }, 0, .none, .uintr }, - - .{ .tpause, .m, .{ .r32 }, .{ 0x66, 0x0f, 0xae }, 6, .none, .waitpkg }, - - .{ .ud0, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xff }, 0, .none, .none }, - .{ .ud1, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xb9 }, 0, .none, .none }, - .{ .ud2, .z, .{}, .{ 0x0f, 0x0b }, 0, .none, .none }, - - .{ .uiret, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xec }, 0, .none, .uintr }, - - .{ .umonitor, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xae }, 6, .none, .waitpkg }, - - .{ .umwait, .m, .{ .r32 }, .{ 0xf2, 0x0f, 0xae }, 6, .none, .waitpkg }, - - .{ .verr, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 4, .none, .none }, - .{ .verw, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 5, .none, .none }, - - .{ .wrfsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 2, .none, .fsgsbase }, - .{ .wrfsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 2, .long, .fsgsbase }, - .{ .wrgsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 3, .none, .fsgsbase }, - .{ .wrgsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 3, .long, .fsgsbase }, - - .{ .wrmsr, .z, .{}, .{ 0x0f, 0x30 }, 0, .none, .none }, - - .{ .wrpkru, .z, .{}, .{ 0x0f, 0x01, 0xef }, 0, .none, .pku }, - - .{ .wrssd, .mr, .{ .m32, .r32 }, .{ 0x0f, 0x38, 0xf6 }, 0, .none, .shstk }, - .{ .wrssq, .mr, .{ .m64, .r64 }, .{ 0x0f, 0x38, 0xf6 }, 0, .long, .shstk }, - - .{ .wrussd, .mr, .{ .m32, .r32 }, .{ 0x66, 0x0f, 0x38, 0xf5 }, 0, .none, .shstk }, - .{ .wrussq, .mr, .{ .m64, .r64 }, .{ 0x66, 0x0f, 0x38, 0xf5 }, 0, .long, .shstk }, - - .{ .xadd, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xc0 }, 0, .none, .none }, - .{ .xadd, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xc0 }, 0, .rex, .none }, - .{ .xadd, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xc1 }, 0, .short, .none }, - .{ .xadd, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xc1 }, 0, .none, .none }, - .{ .xadd, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xc1 }, 0, .long, .none }, - - .{ .xchg, .zo, .{ .ax, .r16 }, .{ 0x90 }, 0, .short, .none }, - .{ .xchg, .oz, .{ .r16, .ax }, .{ 0x90 }, 0, .short, .none }, - .{ .xchg, .zo, .{ .eax, .r32 }, .{ 0x90 }, 0, .none, .none }, - .{ .xchg, .zo, .{ .rax, .r64 }, .{ 0x90 }, 0, .long, .none }, - .{ .xchg, .oz, .{ .r32, .eax }, .{ 0x90 }, 0, .none, .none }, - .{ .xchg, .oz, .{ .r64, .rax }, .{ 0x90 }, 0, .long, .none }, - .{ .xchg, .mr, .{ .rm8, .r8 }, .{ 0x86 }, 0, .none, .none }, - .{ .xchg, .mr, .{ .rm8, .r8 }, .{ 0x86 }, 0, .rex, .none }, - .{ .xchg, .rm, .{ .r8, .rm8 }, .{ 0x86 }, 0, .none, .none }, - .{ .xchg, .rm, .{ .r8, .rm8 }, .{ 0x86 }, 0, .rex, .none }, - .{ .xchg, .mr, .{ .rm16, .r16 }, .{ 0x87 }, 0, .short, .none }, - .{ .xchg, .rm, .{ .r16, .rm16 }, .{ 0x87 }, 0, .short, .none }, - .{ .xchg, .mr, .{ .rm32, .r32 }, .{ 0x87 }, 0, .none, .none }, - .{ .xchg, .mr, .{ .rm64, .r64 }, .{ 0x87 }, 0, .long, .none }, - .{ .xchg, .rm, .{ .r32, .rm32 }, .{ 0x87 }, 0, .none, .none }, - .{ .xchg, .rm, .{ .r64, .rm64 }, .{ 0x87 }, 0, .long, .none }, - - .{ .xgetbv, .z, .{}, .{ 0x0f, 0x01, 0xd0 }, 0, .none, .none }, - - .{ .xlat, .z, .{ .m8 }, .{ 0xd7 }, 0, .none, .@"32bit" }, - .{ .xlat, .z, .{ .m8 }, .{ 0xd7 }, 0, .long, .@"64bit" }, - .{ .xlatb, .z, .{ }, .{ 0xd7 }, 0, .none, .@"32bit" }, - .{ .xlatb, .z, .{ }, .{ 0xd7 }, 0, .long, .@"64bit" }, - - .{ .xor, .zi, .{ .al, .imm8 }, .{ 0x34 }, 0, .none, .none }, - .{ .xor, .zi, .{ .ax, .imm16 }, .{ 0x35 }, 0, .short, .none }, - .{ .xor, .zi, .{ .eax, .imm32 }, .{ 0x35 }, 0, .none, .none }, - .{ .xor, .zi, .{ .rax, .imm32s }, .{ 0x35 }, 0, .long, .none }, - .{ .xor, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 6, .none, .none }, - .{ .xor, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 6, .rex, .none }, - .{ .xor, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 6, .short, .none }, - .{ .xor, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 6, .none, .none }, - .{ .xor, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 6, .long, .none }, - .{ .xor, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 6, .short, .none }, - .{ .xor, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 6, .none, .none }, - .{ .xor, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 6, .long, .none }, - .{ .xor, .mr, .{ .rm8, .r8 }, .{ 0x30 }, 0, .none, .none }, - .{ .xor, .mr, .{ .rm8, .r8 }, .{ 0x30 }, 0, .rex, .none }, - .{ .xor, .mr, .{ .rm16, .r16 }, .{ 0x31 }, 0, .short, .none }, - .{ .xor, .mr, .{ .rm32, .r32 }, .{ 0x31 }, 0, .none, .none }, - .{ .xor, .mr, .{ .rm64, .r64 }, .{ 0x31 }, 0, .long, .none }, - .{ .xor, .rm, .{ .r8, .rm8 }, .{ 0x32 }, 0, .none, .none }, - .{ .xor, .rm, .{ .r8, .rm8 }, .{ 0x32 }, 0, .rex, .none }, - .{ .xor, .rm, .{ .r16, .rm16 }, .{ 0x33 }, 0, .short, .none }, - .{ .xor, .rm, .{ .r32, .rm32 }, .{ 0x33 }, 0, .none, .none }, - .{ .xor, .rm, .{ .r64, .rm64 }, .{ 0x33 }, 0, .long, .none }, - - // X87 - .{ .f2xm1, .z, .{}, .{ 0xd9, 0xf0 }, 0, .none, .x87 }, - - .{ .fabs, .z, .{}, .{ 0xd9, 0xe1 }, 0, .none, .x87 }, - - .{ .fadd, .m, .{ .m32 }, .{ 0xd8 }, 0, .none, .x87 }, - .{ .fadd, .m, .{ .m64 }, .{ 0xdc }, 0, .none, .x87 }, - .{ .fadd, .zo, .{ .st0, .st }, .{ 0xd8, 0xc0 }, 0, .none, .x87 }, - .{ .fadd, .oz, .{ .st, .st0 }, .{ 0xdc, 0xc0 }, 0, .none, .x87 }, - .{ .faddp, .oz, .{ .st, .st0 }, .{ 0xde, 0xc0 }, 0, .none, .x87 }, - .{ .faddp, .z, .{ }, .{ 0xde, 0xc1 }, 0, .none, .x87 }, - .{ .fiadd, .m, .{ .m32 }, .{ 0xda }, 0, .none, .x87 }, - .{ .fiadd, .m, .{ .m16 }, .{ 0xde }, 0, .none, .x87 }, - - .{ .fbld, .m, .{ .m80 }, .{ 0xdf }, 4, .none, .x87 }, - - .{ .fbstp, .m, .{ .m80 }, .{ 0xdf }, 6, .none, .x87 }, - - .{ .fchs, .z, .{}, .{ 0xd9, 0xe0 }, 0, .none, .x87 }, - - .{ .fclex, .z, .{}, .{ 0xdb, 0xe2 }, 0, .wait, .x87 }, - .{ .fnclex, .z, .{}, .{ 0xdb, 0xe2 }, 0, .none, .x87 }, - - .{ .fcmovb, .zo, .{ .st0, .st }, .{ 0xda, 0xc0 }, 0, .none, .@"cmov x87" }, - .{ .fcmove, .zo, .{ .st0, .st }, .{ 0xda, 0xc8 }, 0, .none, .@"cmov x87" }, - .{ .fcmovbe, .zo, .{ .st0, .st }, .{ 0xda, 0xd0 }, 0, .none, .@"cmov x87" }, - .{ .fcmovu, .zo, .{ .st0, .st }, .{ 0xda, 0xd8 }, 0, .none, .@"cmov x87" }, - .{ .fcmovnb, .zo, .{ .st0, .st }, .{ 0xdb, 0xc0 }, 0, .none, .@"cmov x87" }, - .{ .fcmovne, .zo, .{ .st0, .st }, .{ 0xdb, 0xc8 }, 0, .none, .@"cmov x87" }, - .{ .fcmovnbe, .zo, .{ .st0, .st }, .{ 0xdb, 0xd0 }, 0, .none, .@"cmov x87" }, - .{ .fcmovnu, .zo, .{ .st0, .st }, .{ 0xdb, 0xd8 }, 0, .none, .@"cmov x87" }, - - .{ .fcom, .m, .{ .m32 }, .{ 0xd8 }, 2, .none, .x87 }, - .{ .fcom, .m, .{ .m64 }, .{ 0xdc }, 2, .none, .x87 }, - .{ .fcom, .o, .{ .st }, .{ 0xd8, 0xd0 }, 0, .none, .x87 }, - .{ .fcom, .z, .{ }, .{ 0xd8, 0xd1 }, 0, .none, .x87 }, - .{ .fcomp, .m, .{ .m32 }, .{ 0xd8 }, 3, .none, .x87 }, - .{ .fcomp, .m, .{ .m64 }, .{ 0xdc }, 3, .none, .x87 }, - .{ .fcomp, .o, .{ .st }, .{ 0xd8, 0xd8 }, 0, .none, .x87 }, - .{ .fcomp, .z, .{ }, .{ 0xd8, 0xd9 }, 0, .none, .x87 }, - .{ .fcompp, .z, .{ }, .{ 0xde, 0xd9 }, 0, .none, .x87 }, - - .{ .fcomi, .zo, .{ .st0, .st }, .{ 0xdb, 0xf0 }, 0, .none, .@"cmov x87" }, - .{ .fcomip, .zo, .{ .st0, .st }, .{ 0xdf, 0xf0 }, 0, .none, .@"cmov x87" }, - .{ .fucomi, .zo, .{ .st0, .st }, .{ 0xdb, 0xe8 }, 0, .none, .@"cmov x87" }, - .{ .fucomip, .zo, .{ .st0, .st }, .{ 0xdf, 0xe8 }, 0, .none, .@"cmov x87" }, - - .{ .fcos, .z, .{}, .{ 0xd9, 0xff }, 0, .none, .x87 }, - - .{ .fdecstp, .z, .{}, .{ 0xd9, 0xf6 }, 0, .none, .x87 }, - - .{ .fdiv, .m, .{ .m32 }, .{ 0xd8 }, 6, .none, .x87 }, - .{ .fdiv, .m, .{ .m64 }, .{ 0xdc }, 6, .none, .x87 }, - .{ .fdiv, .zo, .{ .st0, .st }, .{ 0xd8, 0xf0 }, 0, .none, .x87 }, - .{ .fdiv, .oz, .{ .st, .st0 }, .{ 0xdc, 0xf8 }, 0, .none, .x87 }, - .{ .fdivp, .oz, .{ .st, .st0 }, .{ 0xde, 0xf8 }, 0, .none, .x87 }, - .{ .fdivp, .z, .{ }, .{ 0xde, 0xf9 }, 0, .none, .x87 }, - .{ .fidiv, .m, .{ .m32 }, .{ 0xda }, 6, .none, .x87 }, - .{ .fidiv, .m, .{ .m16 }, .{ 0xde }, 6, .none, .x87 }, - - .{ .fdivr, .m, .{ .m32 }, .{ 0xd8 }, 7, .none, .x87 }, - .{ .fdivr, .m, .{ .m64 }, .{ 0xdc }, 7, .none, .x87 }, - .{ .fdivr, .zo, .{ .st0, .st }, .{ 0xd8, 0xf8 }, 0, .none, .x87 }, - .{ .fdivr, .oz, .{ .st, .st0 }, .{ 0xdc, 0xf0 }, 0, .none, .x87 }, - .{ .fdivrp, .oz, .{ .st, .st0 }, .{ 0xde, 0xf0 }, 0, .none, .x87 }, - .{ .fdivrp, .z, .{ }, .{ 0xde, 0xf1 }, 0, .none, .x87 }, - .{ .fidivr, .m, .{ .m32 }, .{ 0xda }, 7, .none, .x87 }, - .{ .fidivr, .m, .{ .m16 }, .{ 0xde }, 7, .none, .x87 }, - - .{ .ffree, .o, .{ .st }, .{ 0xdd, 0xc0 }, 0, .none, .x87 }, - - .{ .ficom, .m, .{ .m16 }, .{ 0xde }, 2, .none, .x87 }, - .{ .ficom, .m, .{ .m32 }, .{ 0xda }, 2, .none, .x87 }, - .{ .ficomp, .m, .{ .m16 }, .{ 0xde }, 3, .none, .x87 }, - .{ .ficomp, .m, .{ .m32 }, .{ 0xda }, 3, .none, .x87 }, - - .{ .fild, .m, .{ .m16 }, .{ 0xdf }, 0, .none, .x87 }, - .{ .fild, .m, .{ .m32 }, .{ 0xdb }, 0, .none, .x87 }, - .{ .fild, .m, .{ .m64 }, .{ 0xdf }, 5, .none, .x87 }, - - .{ .fincstp, .z, .{}, .{ 0xd9, 0xf7 }, 0, .none, .x87 }, - - .{ .finit, .z, .{}, .{ 0xdb, 0xe3 }, 0, .wait, .x87 }, - .{ .fninit, .z, .{}, .{ 0xdb, 0xe3 }, 0, .none, .x87 }, - - .{ .fist, .m, .{ .m16 }, .{ 0xdf }, 2, .none, .x87 }, - .{ .fist, .m, .{ .m32 }, .{ 0xdb }, 2, .none, .x87 }, - .{ .fistp, .m, .{ .m16 }, .{ 0xdf }, 3, .none, .x87 }, - .{ .fistp, .m, .{ .m32 }, .{ 0xdb }, 3, .none, .x87 }, - .{ .fistp, .m, .{ .m64 }, .{ 0xdf }, 7, .none, .x87 }, - - .{ .fld, .m, .{ .m32 }, .{ 0xd9 }, 0, .none, .x87 }, - .{ .fld, .m, .{ .m64 }, .{ 0xdd }, 0, .none, .x87 }, - .{ .fld, .m, .{ .m80 }, .{ 0xdb }, 5, .none, .x87 }, - .{ .fld, .o, .{ .st }, .{ 0xd9, 0xc0 }, 0, .none, .x87 }, - - .{ .fld1, .z, .{}, .{ 0xd9, 0xe8 }, 0, .none, .x87 }, - .{ .fldl2t, .z, .{}, .{ 0xd9, 0xe9 }, 0, .none, .x87 }, - .{ .fldl2e, .z, .{}, .{ 0xd9, 0xea }, 0, .none, .x87 }, - .{ .fldpi, .z, .{}, .{ 0xd9, 0xeb }, 0, .none, .x87 }, - .{ .fldlg2, .z, .{}, .{ 0xd9, 0xec }, 0, .none, .x87 }, - .{ .fldln2, .z, .{}, .{ 0xd9, 0xed }, 0, .none, .x87 }, - .{ .fldz, .z, .{}, .{ 0xd9, 0xee }, 0, .none, .x87 }, - - .{ .fldcw, .m, .{ .m16 }, .{ 0xd9 }, 5, .none, .x87 }, - - .{ .fldenv, .m, .{ .m }, .{ 0xd9 }, 4, .none, .x87 }, - - .{ .fmul, .m, .{ .m32 }, .{ 0xd8 }, 1, .none, .x87 }, - .{ .fmul, .m, .{ .m64 }, .{ 0xdc }, 1, .none, .x87 }, - .{ .fmul, .zo, .{ .st0, .st }, .{ 0xd8, 0xc8 }, 0, .none, .x87 }, - .{ .fmul, .oz, .{ .st, .st0 }, .{ 0xdc, 0xc8 }, 0, .none, .x87 }, - .{ .fmulp, .oz, .{ .st, .st0 }, .{ 0xde, 0xc8 }, 0, .none, .x87 }, - .{ .fmulp, .z, .{ }, .{ 0xde, 0xc9 }, 0, .none, .x87 }, - .{ .fimul, .m, .{ .m32 }, .{ 0xda }, 1, .none, .x87 }, - .{ .fimul, .m, .{ .m16 }, .{ 0xde }, 1, .none, .x87 }, - - .{ .fnop, .z, .{}, .{ 0xd9, 0xd0 }, 0, .none, .x87 }, - - .{ .fpatan, .z, .{}, .{ 0xd9, 0xf3 }, 0, .none, .x87 }, - - .{ .fprem, .z, .{}, .{ 0xd9, 0xf8 }, 0, .none, .x87 }, - - .{ .fprem1, .z, .{}, .{ 0xd9, 0xf5 }, 0, .none, .x87 }, - - .{ .fptan, .z, .{}, .{ 0xd9, 0xf2 }, 0, .none, .x87 }, - - .{ .frndint, .z, .{}, .{ 0xd9, 0xfc }, 0, .none, .x87 }, - - .{ .frstor, .m, .{ .m }, .{ 0xdd }, 4, .none, .x87 }, - - .{ .fsave, .m, .{ .m }, .{ 0xdd }, 6, .wait, .x87 }, - .{ .fnsave, .m, .{ .m }, .{ 0xdd }, 6, .none, .x87 }, - - .{ .fscale, .z, .{}, .{ 0xd9, 0xfd }, 0, .none, .x87 }, - - .{ .fsin, .z, .{}, .{ 0xd9, 0xfe }, 0, .none, .x87 }, - - .{ .fsincos, .z, .{}, .{ 0xd9, 0xfb }, 0, .none, .x87 }, - - .{ .fsqrt, .z, .{}, .{ 0xd9, 0xfa }, 0, .none, .x87 }, - - .{ .fst, .m, .{ .m32 }, .{ 0xd9 }, 2, .none, .x87 }, - .{ .fst, .m, .{ .m64 }, .{ 0xdd }, 2, .none, .x87 }, - .{ .fst, .o, .{ .st }, .{ 0xdd, 0xd0 }, 0, .none, .x87 }, - .{ .fstp, .m, .{ .m32 }, .{ 0xd9 }, 3, .none, .x87 }, - .{ .fstp, .m, .{ .m64 }, .{ 0xdd }, 3, .none, .x87 }, - .{ .fstp, .m, .{ .m80 }, .{ 0xdb }, 7, .none, .x87 }, - .{ .fstp, .o, .{ .st }, .{ 0xdd, 0xd8 }, 0, .none, .x87 }, - - .{ .fstcw, .m, .{ .m16 }, .{ 0xd9 }, 7, .wait, .x87 }, - .{ .fnstcw, .m, .{ .m16 }, .{ 0xd9 }, 7, .none, .x87 }, - - .{ .fstenv, .m, .{ .m }, .{ 0xd9 }, 6, .wait, .x87 }, - .{ .fnstenv, .m, .{ .m }, .{ 0xd9 }, 6, .none, .x87 }, - - .{ .fstsw, .m, .{ .m16 }, .{ 0xdd }, 7, .wait, .x87 }, - .{ .fstsw, .m, .{ .ax }, .{ 0xdf }, 4, .wait, .x87 }, - .{ .fnstsw, .m, .{ .m16 }, .{ 0xdd }, 7, .none, .x87 }, - .{ .fnstsw, .m, .{ .ax }, .{ 0xdf }, 4, .none, .x87 }, - - .{ .fsub, .m, .{ .m32 }, .{ 0xd8 }, 4, .none, .x87 }, - .{ .fsub, .m, .{ .m64 }, .{ 0xdc }, 4, .none, .x87 }, - .{ .fsub, .zo, .{ .st0, .st }, .{ 0xd8, 0xe0 }, 0, .none, .x87 }, - .{ .fsub, .oz, .{ .st, .st0 }, .{ 0xdc, 0xe8 }, 0, .none, .x87 }, - .{ .fsubp, .oz, .{ .st, .st0 }, .{ 0xde, 0xe8 }, 0, .none, .x87 }, - .{ .fsubp, .z, .{ }, .{ 0xde, 0xe9 }, 0, .none, .x87 }, - .{ .fisub, .m, .{ .m32 }, .{ 0xda }, 4, .none, .x87 }, - .{ .fisub, .m, .{ .m16 }, .{ 0xde }, 4, .none, .x87 }, - - .{ .fsubr, .m, .{ .m32 }, .{ 0xd8 }, 5, .none, .x87 }, - .{ .fsubr, .m, .{ .m64 }, .{ 0xdc }, 5, .none, .x87 }, - .{ .fsubr, .zo, .{ .st0, .st }, .{ 0xd8, 0xe8 }, 0, .none, .x87 }, - .{ .fsubr, .oz, .{ .st, .st0 }, .{ 0xdc, 0xe0 }, 0, .none, .x87 }, - .{ .fsubrp, .oz, .{ .st, .st0 }, .{ 0xde, 0xe0 }, 0, .none, .x87 }, - .{ .fsubrp, .z, .{ }, .{ 0xde, 0xe1 }, 0, .none, .x87 }, - .{ .fisubr, .m, .{ .m32 }, .{ 0xda }, 5, .none, .x87 }, - .{ .fisubr, .m, .{ .m16 }, .{ 0xde }, 5, .none, .x87 }, - - .{ .ftst, .z, .{}, .{ 0xd9, 0xe4 }, 0, .none, .x87 }, - - .{ .fucom, .o, .{ .st }, .{ 0xdd, 0xe0 }, 0, .none, .x87 }, - .{ .fucom, .z, .{ }, .{ 0xdd, 0xe1 }, 0, .none, .x87 }, - .{ .fucomp, .o, .{ .st }, .{ 0xdd, 0xe8 }, 0, .none, .x87 }, - .{ .fucomp, .z, .{ }, .{ 0xdd, 0xe9 }, 0, .none, .x87 }, - .{ .fucompp, .z, .{ }, .{ 0xda, 0xe9 }, 0, .none, .x87 }, - - .{ .fxam, .z, .{}, .{ 0xd9, 0xe5 }, 0, .none, .x87 }, - - .{ .fxch, .o, .{ .st }, .{ 0xd9, 0xc8 }, 0, .none, .x87 }, - .{ .fxch, .z, .{ }, .{ 0xd9, 0xc9 }, 0, .none, .x87 }, - - .{ .fxtract, .z, .{}, .{ 0xd9, 0xf4 }, 0, .none, .x87 }, - - .{ .fyl2x, .z, .{}, .{ 0xd9, 0xf1 }, 0, .none, .x87 }, - - .{ .fyl2xp1, .z, .{}, .{ 0xd9, 0xf9 }, 0, .none, .x87 }, - - .{ .wait, .z, .{}, .{ 0x9b }, 0, .none, .x87 }, - .{ .fwait, .z, .{}, .{ 0x9b }, 0, .none, .x87 }, - - // MMX - .{ .emms, .z, .{}, .{ 0x0f, 0x77 }, 0, .none, .mmx }, - - // SSE - .{ .addps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x58 }, 0, .none, .sse }, - - .{ .addss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse }, - - .{ .andnps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x55 }, 0, .none, .sse }, - - .{ .andps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x54 }, 0, .none, .sse }, - - .{ .cmpps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc2 }, 0, .none, .sse }, - - .{ .cmpss, .rmi, .{ .xmm, .xmm_m32, .imm8 }, .{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse }, - - .{ .comiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2f }, 0, .none, .sse }, - - .{ .cvtpi2ps, .rm, .{ .xmm, .mm_m64 }, .{ 0x0f, 0x2a }, 0, .none, .sse }, - - .{ .cvtps2pi, .rm, .{ .mm, .xmm_m64 }, .{ 0x0f, 0x2d }, 0, .none, .sse }, - - .{ .cvtsi2ss, .rm, .{ .xmm, .rm32 }, .{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse }, - .{ .cvtsi2ss, .rm, .{ .xmm, .rm64 }, .{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse }, - - .{ .cvtss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .none, .sse }, - .{ .cvtss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .long, .sse }, - - .{ .cvttps2pi, .rm, .{ .mm, .xmm_m64 }, .{ 0x0f, 0x2c }, 0, .none, .sse }, - - .{ .cvttss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .none, .sse }, - .{ .cvttss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .long, .sse }, - - .{ .divps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5e }, 0, .none, .sse }, - - .{ .divss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse }, - - .{ .fxrstor, .m, .{ .m }, .{ 0x0f, 0xae }, 1, .none, .fxsr }, - .{ .fxrstor64, .m, .{ .m }, .{ 0x0f, 0xae }, 1, .long, .fxsr }, - - .{ .fxsave, .m, .{ .m }, .{ 0x0f, 0xae }, 0, .none, .fxsr }, - .{ .fxsave64, .m, .{ .m }, .{ 0x0f, 0xae }, 0, .long, .fxsr }, - - .{ .ldmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 2, .none, .sse }, - - .{ .maxps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5f }, 0, .none, .sse }, - - .{ .maxss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5f }, 0, .none, .sse }, - - .{ .minps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5d }, 0, .none, .sse }, - - .{ .minss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5d }, 0, .none, .sse }, - - .{ .movaps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x28 }, 0, .none, .sse }, - .{ .movaps, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x29 }, 0, .none, .sse }, - - .{ .movhlps, .rm, .{ .xmm, .xmm }, .{ 0x0f, 0x12 }, 0, .none, .sse }, - - .{ .movhps, .rm, .{ .xmm, .m64 }, .{ 0x0f, 0x16 }, 0, .none, .sse }, - .{ .movhps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x17 }, 0, .none, .sse }, - - .{ .movlhps, .rm, .{ .xmm, .xmm }, .{ 0x0f, 0x16 }, 0, .none, .sse }, - - .{ .movlps, .rm, .{ .xmm, .m64 }, .{ 0x0f, 0x12 }, 0, .none, .sse }, - .{ .movlps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x13 }, 0, .none, .sse }, - - .{ .movmskps, .rm, .{ .r32, .xmm }, .{ 0x0f, 0x50 }, 0, .none, .sse }, - .{ .movmskps, .rm, .{ .r64, .xmm }, .{ 0x0f, 0x50 }, 0, .none, .sse }, - - .{ .movss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse }, - .{ .movss, .mr, .{ .xmm_m32, .xmm }, .{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse }, - - .{ .movups, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x10 }, 0, .none, .sse }, - .{ .movups, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x11 }, 0, .none, .sse }, - - .{ .mulps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x59 }, 0, .none, .sse }, - - .{ .mulss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x59 }, 0, .none, .sse }, - - .{ .orps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x56 }, 0, .none, .sse }, - - .{ .pmovmskb, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse }, - .{ .pmovmskb, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse }, - - .{ .prefetchit0, .m, .{ .mrip8 }, .{ 0x0f, 0x18 }, 7, .none, .@"prefetchi 64bit" }, - .{ .prefetchit1, .m, .{ .mrip8 }, .{ 0x0f, 0x18 }, 6, .none, .@"prefetchi 64bit" }, - - .{ .prefetchnta, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 0, .none, .prefetch }, - .{ .prefetcht0, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 1, .none, .prefetch }, - .{ .prefetcht1, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 2, .none, .prefetch }, - .{ .prefetcht2, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 3, .none, .prefetch }, - - .{ .prefetchw, .m, .{ .m8 }, .{ 0x0f, 0x0d }, 1, .none, .prfchw }, - - .{ .prefetchwt1, .m, .{ .m8 }, .{ 0x0f, 0x0d }, 2, .none, .prefetchwt1 }, - - .{ .shufps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc6 }, 0, .none, .sse }, - - .{ .sqrtps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x51 }, 0, .none, .sse }, - - .{ .sqrtss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse }, - - .{ .stmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 3, .none, .sse }, - - .{ .subps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5c }, 0, .none, .sse }, - - .{ .subss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse }, - - .{ .ucomiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2e }, 0, .none, .sse }, - - .{ .unpckhps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x15 }, 0, .none, .sse }, - - .{ .unpcklps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x14 }, 0, .none, .sse }, - - .{ .xorps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x57 }, 0, .none, .sse }, - - // SSE2 - .{ .addpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x58 }, 0, .none, .sse2 }, - - .{ .addsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x58 }, 0, .none, .sse2 }, - - .{ .andnpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x55 }, 0, .none, .sse2 }, - - .{ .andpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 }, - - .{ .cmppd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc2 }, 0, .none, .sse2 }, - - .{ .cmpsd, .rmi, .{ .xmm, .xmm_m64, .imm8 }, .{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 }, - - .{ .comisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2f }, 0, .none, .sse2 }, - - .{ .cvtdq2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0xe6 }, 0, .none, .sse2 }, - - .{ .cvtdq2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5b }, 0, .none, .sse2 }, - - .{ .cvtpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xe6 }, 0, .none, .sse2 }, - - .{ .cvtpd2pi, .rm, .{ .mm, .xmm_m128 }, .{ 0x66, 0x0f, 0x2d }, 0, .none, .sse2 }, - - .{ .cvtpd2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5a }, 0, .none, .sse2 }, - - .{ .cvtpi2pd, .rm, .{ .xmm, .mm_m64 }, .{ 0x66, 0x0f, 0x2a }, 0, .none, .sse2 }, - - .{ .cvtps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5b }, 0, .none, .sse2 }, - - .{ .cvtps2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x0f, 0x5a }, 0, .none, .sse2 }, - - .{ .cvtsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .none, .sse2 }, - .{ .cvtsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .long, .sse2 }, - - .{ .cvtsd2ss, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5a }, 0, .none, .sse2 }, - - .{ .cvtsi2sd, .rm, .{ .xmm, .rm32 }, .{ 0xf2, 0x0f, 0x2a }, 0, .none, .sse2 }, - .{ .cvtsi2sd, .rm, .{ .xmm, .rm64 }, .{ 0xf2, 0x0f, 0x2a }, 0, .long, .sse2 }, - - .{ .cvtss2sd, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 }, - - .{ .cvttpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe6 }, 0, .none, .sse2 }, - - .{ .cvttpd2pi, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x2c }, 0, .none, .sse2 }, - - .{ .cvttps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x5b }, 0, .none, .sse2 }, - - .{ .cvttsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .none, .sse2 }, - .{ .cvttsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .long, .sse2 }, - - .{ .divpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5e }, 0, .none, .sse2 }, - - .{ .divsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 }, - - .{ .gf2p8affineinvqb, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xcf }, 0, .none, .gfni }, - - .{ .gf2p8affineqb, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xce }, 0, .none, .gfni }, - - .{ .gf2p8mulb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xcf }, 0, .none, .gfni }, - - .{ .maxpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5f }, 0, .none, .sse2 }, - - .{ .maxsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5f }, 0, .none, .sse2 }, - - .{ .minpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5d }, 0, .none, .sse2 }, - - .{ .minsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5d }, 0, .none, .sse2 }, - - .{ .movapd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x28 }, 0, .none, .sse2 }, - .{ .movapd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x29 }, 0, .none, .sse2 }, - - .{ .movd, .rm, .{ .xmm, .rm32 }, .{ 0x66, 0x0f, 0x6e }, 0, .none, .sse2 }, - .{ .movq, .rm, .{ .xmm, .rm64 }, .{ 0x66, 0x0f, 0x6e }, 0, .long, .sse2 }, - .{ .movd, .mr, .{ .rm32, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 }, - .{ .movq, .mr, .{ .rm64, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .long, .sse2 }, - - .{ .movdqa, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6f }, 0, .none, .sse2 }, - .{ .movdqa, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x7f }, 0, .none, .sse2 }, - - .{ .movdqu, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 }, - .{ .movdqu, .mr, .{ .xmm_m128, .xmm }, .{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 }, - - .{ .movhpd, .rm, .{ .xmm, .m64 }, .{ 0x66, 0x0f, 0x16 }, 0, .none, .sse2 }, - .{ .movhpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x17 }, 0, .none, .sse2 }, - - .{ .movlpd, .rm, .{ .xmm, .m64 }, .{ 0x66, 0x0f, 0x12 }, 0, .none, .sse2 }, - .{ .movlpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x13 }, 0, .none, .sse2 }, - - .{ .movmskpd, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 }, - .{ .movmskpd, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 }, - - .{ .movsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x10 }, 0, .none, .sse2 }, - .{ .movsd, .mr, .{ .xmm_m64, .xmm }, .{ 0xf2, 0x0f, 0x11 }, 0, .none, .sse2 }, - - .{ .movq, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0x7e }, 0, .none, .sse2 }, - .{ .movq, .mr, .{ .xmm_m64, .xmm }, .{ 0x66, 0x0f, 0xd6 }, 0, .none, .sse2 }, - - .{ .movupd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x10 }, 0, .none, .sse2 }, - .{ .movupd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x11 }, 0, .none, .sse2 }, - - .{ .mulpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x59 }, 0, .none, .sse2 }, - - .{ .mulsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x59 }, 0, .none, .sse2 }, - - .{ .orpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 }, - - .{ .packsswb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x63 }, 0, .none, .sse2 }, - .{ .packssdw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6b }, 0, .none, .sse2 }, - - .{ .packuswb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x67 }, 0, .none, .sse2 }, - - .{ .paddb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfc }, 0, .none, .sse2 }, - .{ .paddw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfd }, 0, .none, .sse2 }, - .{ .paddd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfe }, 0, .none, .sse2 }, - .{ .paddq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd4 }, 0, .none, .sse2 }, - - .{ .paddsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xec }, 0, .none, .sse2 }, - .{ .paddsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xed }, 0, .none, .sse2 }, - - .{ .paddusb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdc }, 0, .none, .sse2 }, - .{ .paddusw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdd }, 0, .none, .sse2 }, - - .{ .pand, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdb }, 0, .none, .sse2 }, - - .{ .pandn, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdf }, 0, .none, .sse2 }, - - .{ .pcmpeqb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x74 }, 0, .none, .sse2 }, - .{ .pcmpeqw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x75 }, 0, .none, .sse2 }, - .{ .pcmpeqd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x76 }, 0, .none, .sse2 }, - - .{ .pcmpgtb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x64 }, 0, .none, .sse2 }, - .{ .pcmpgtw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x65 }, 0, .none, .sse2 }, - .{ .pcmpgtd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x66 }, 0, .none, .sse2 }, - - .{ .pextrw, .rmi, .{ .r32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 }, - - .{ .pinsrw, .rmi, .{ .xmm, .r32_m16, .imm8 }, .{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 }, - - .{ .pmaddwd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf5 }, 0, .none, .sse2 }, - - .{ .pmaxsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xee }, 0, .none, .sse2 }, - - .{ .pmaxub, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xde }, 0, .none, .sse2 }, - - .{ .pminsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xea }, 0, .none, .sse2 }, - - .{ .pminub, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xda }, 0, .none, .sse2 }, - - .{ .pmulhuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe4 }, 0, .none, .sse2 }, - - .{ .pmulhw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe5 }, 0, .none, .sse2 }, - - .{ .pmullw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd5 }, 0, .none, .sse2 }, - - .{ .pmuludq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf4 }, 0, .none, .sse2 }, - - .{ .por, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xeb }, 0, .none, .sse2 }, - - .{ .psadbw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf6 }, 0, .none, .sse2 }, - - .{ .pshufd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .none, .sse2 }, - - .{ .pshufhw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf3, 0x0f, 0x70 }, 0, .none, .sse2 }, - - .{ .pshuflw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 }, - - .{ .psllw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf1 }, 0, .none, .sse2 }, - .{ .psllw, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 6, .none, .sse2 }, - .{ .pslld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf2 }, 0, .none, .sse2 }, - .{ .pslld, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 6, .none, .sse2 }, - .{ .psllq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf3 }, 0, .none, .sse2 }, - .{ .psllq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 6, .none, .sse2 }, - - .{ .pslldq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 7, .none, .sse2 }, - - .{ .psraw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe1 }, 0, .none, .sse2 }, - .{ .psraw, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 4, .none, .sse2 }, - .{ .psrad, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe2 }, 0, .none, .sse2 }, - .{ .psrad, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 4, .none, .sse2 }, - - .{ .psrlw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 }, - .{ .psrlw, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 }, - .{ .psrld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd2 }, 0, .none, .sse2 }, - .{ .psrld, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 2, .none, .sse2 }, - .{ .psrlq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 }, - .{ .psrlq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 }, - - .{ .psrldq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 3, .none, .sse2 }, - - .{ .psubb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf8 }, 0, .none, .sse2 }, - .{ .psubw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf9 }, 0, .none, .sse2 }, - .{ .psubd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfa }, 0, .none, .sse2 }, - - .{ .psubsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe8 }, 0, .none, .sse2 }, - .{ .psubsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe9 }, 0, .none, .sse2 }, - - .{ .psubq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfb }, 0, .none, .sse2 }, - - .{ .psubusb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd8 }, 0, .none, .sse2 }, - .{ .psubusw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd9 }, 0, .none, .sse2 }, - - .{ .punpckhbw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 }, - .{ .punpckhwd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 }, - .{ .punpckhdq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 }, - .{ .punpckhqdq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6d }, 0, .none, .sse2 }, - - .{ .punpcklbw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 }, - .{ .punpcklwd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 }, - .{ .punpckldq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 }, - .{ .punpcklqdq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 }, - - .{ .pxor, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xef }, 0, .none, .sse2 }, - - .{ .shufpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc6 }, 0, .none, .sse2 }, - - .{ .sqrtpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 }, - - .{ .sqrtsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 }, - - .{ .subpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5c }, 0, .none, .sse2 }, - - .{ .subsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 }, - - .{ .ucomisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2e }, 0, .none, .sse2 }, - - .{ .unpckhpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x15 }, 0, .none, .sse2 }, - - .{ .unpcklpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x14 }, 0, .none, .sse2 }, - - .{ .xorpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x57 }, 0, .none, .sse2 }, - - // SSE3 - .{ .addsubpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd0 }, 0, .none, .sse3 }, - - .{ .addsubps, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .none, .sse3 }, - - .{ .fisttp, .m, .{ .m16 }, .{ 0xdf }, 1, .none, .@"sse3 x87" }, - .{ .fisttp, .m, .{ .m32 }, .{ 0xdb }, 1, .none, .@"sse3 x87" }, - .{ .fisttp, .m, .{ .m64 }, .{ 0xdd }, 1, .none, .@"sse3 x87" }, - - .{ .haddpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x7c }, 0, .none, .sse3 }, - - .{ .haddps, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0x7c }, 0, .none, .sse3 }, - - .{ .lddqu, .rm, .{ .xmm, .m128 }, .{ 0xf2, 0x0f, 0xf0 }, 0, .none, .sse3 }, - - .{ .movddup, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x12 }, 0, .none, .sse3 }, - - .{ .movshdup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x16 }, 0, .none, .sse3 }, - - .{ .movsldup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 }, - - // SSSE3 - .{ .pabsb, .rm, .{ .mm, .mm_m64 }, .{ 0x0f, 0x38, 0x1c }, 0, .none, .ssse3 }, - .{ .pabsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1c }, 0, .none, .ssse3 }, - .{ .pabsd, .rm, .{ .mm, .mm_m64 }, .{ 0x0f, 0x38, 0x1e }, 0, .none, .ssse3 }, - .{ .pabsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1e }, 0, .none, .ssse3 }, - .{ .pabsw, .rm, .{ .mm, .mm_m64 }, .{ 0x0f, 0x38, 0x1d }, 0, .none, .ssse3 }, - .{ .pabsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1d }, 0, .none, .ssse3 }, - - .{ .palignr, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 }, - - .{ .phaddw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x01 }, 0, .none, .ssse3 }, - .{ .phaddd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x02 }, 0, .none, .ssse3 }, - - .{ .phaddsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x03 }, 0, .none, .ssse3 }, - - .{ .phsubw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x05 }, 0, .none, .ssse3 }, - .{ .phsubd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x06 }, 0, .none, .ssse3 }, - - .{ .phsubsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x07 }, 0, .none, .ssse3 }, - - .{ .pmaddubsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x04 }, 0, .none, .ssse3 }, - - .{ .pmulhrsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0b }, 0, .none, .ssse3 }, - - .{ .pshufb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 }, - - .{ .psignb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x08 }, 0, .none, .ssse3 }, - .{ .psignw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x09 }, 0, .none, .ssse3 }, - .{ .psignd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0a }, 0, .none, .ssse3 }, - - // SSE4.1 - .{ .blendpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 }, - - .{ .blendps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .none, .sse4_1 }, - - .{ .blendvpd, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x15 }, 0, .none, .sse4_1 }, - .{ .blendvpd, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x66, 0x0f, 0x38, 0x15 }, 0, .none, .sse4_1 }, - - .{ .blendvps, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x14 }, 0, .none, .sse4_1 }, - .{ .blendvps, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x66, 0x0f, 0x38, 0x14 }, 0, .none, .sse4_1 }, - - .{ .dppd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x41 }, 0, .none, .sse4_1 }, - - .{ .dpps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x40 }, 0, .none, .sse4_1 }, - - .{ .extractps, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .none, .sse4_1 }, - - .{ .insertps, .rmi, .{ .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .none, .sse4_1 }, - - .{ .packusdw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 }, - - .{ .pblendvb, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 }, - .{ .pblendvb, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 }, - - .{ .pblendw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .none, .sse4_1 }, - - .{ .pcmpeqq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 }, - - .{ .pextrb, .mri, .{ .r32_m8, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 }, - .{ .pextrd, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 }, - .{ .pextrq, .mri, .{ .rm64, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 }, - - .{ .pextrw, .mri, .{ .r32_m16, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 }, - - .{ .phminposuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x41 }, 0, .none, .sse4_1 }, - - .{ .pinsrb, .rmi, .{ .xmm, .r32_m8, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .none, .sse4_1 }, - .{ .pinsrd, .rmi, .{ .xmm, .rm32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 }, - .{ .pinsrq, .rmi, .{ .xmm, .rm64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 }, - - .{ .pmaxsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3c }, 0, .none, .sse4_1 }, - .{ .pmaxsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3d }, 0, .none, .sse4_1 }, - - .{ .pmaxuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3e }, 0, .none, .sse4_1 }, - - .{ .pmaxud, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3f }, 0, .none, .sse4_1 }, - - .{ .pminsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x38 }, 0, .none, .sse4_1 }, - .{ .pminsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x39 }, 0, .none, .sse4_1 }, - - .{ .pminuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3a }, 0, .none, .sse4_1 }, - - .{ .pminud, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 }, - - .{ .pmovsxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x20 }, 0, .none, .sse4_1 }, - .{ .pmovsxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x21 }, 0, .none, .sse4_1 }, - .{ .pmovsxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x22 }, 0, .none, .sse4_1 }, - .{ .pmovsxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x23 }, 0, .none, .sse4_1 }, - .{ .pmovsxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x24 }, 0, .none, .sse4_1 }, - .{ .pmovsxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x25 }, 0, .none, .sse4_1 }, - - .{ .pmovzxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x30 }, 0, .none, .sse4_1 }, - .{ .pmovzxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x31 }, 0, .none, .sse4_1 }, - .{ .pmovzxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x32 }, 0, .none, .sse4_1 }, - .{ .pmovzxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x33 }, 0, .none, .sse4_1 }, - .{ .pmovzxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 }, - .{ .pmovzxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 }, - - .{ .pmuldq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .none, .sse4_1 }, - - .{ .pmulld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 }, - - .{ .ptest, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .none, .sse4_1 }, - - .{ .roundpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 }, - - .{ .roundps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 }, - - .{ .roundsd, .rmi, .{ .xmm, .xmm_m64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 }, - - .{ .roundss, .rmi, .{ .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 }, - - // SSE4.2 - .{ .crc32, .rm, .{ .r32, .rm8 }, .{ 0xf2, 0x0f, 0x38, 0xf0 }, 0, .none, .crc32 }, - .{ .crc32, .rm, .{ .r32, .rm8 }, .{ 0xf2, 0x0f, 0x38, 0xf0 }, 0, .rex, .crc32 }, - .{ .crc32, .rm, .{ .r32, .rm16 }, .{ 0xf2, 0x0f, 0x38, 0xf1 }, 0, .short, .crc32 }, - .{ .crc32, .rm, .{ .r32, .rm32 }, .{ 0xf2, 0x0f, 0x38, 0xf1 }, 0, .none, .crc32 }, - .{ .crc32, .rm, .{ .r64, .rm8 }, .{ 0xf2, 0x0f, 0x38, 0xf0 }, 0, .long, .crc32 }, - .{ .crc32, .rm, .{ .r64, .rm64 }, .{ 0xf2, 0x0f, 0x38, 0xf1 }, 0, .long, .crc32 }, - - .{ .pcmpgtq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x37 }, 0, .none, .sse4_2 }, - - // ABM - .{ .lzcnt, .rm, .{ .r16, .rm16 }, .{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt }, - .{ .lzcnt, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt }, - .{ .lzcnt, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt }, - - .{ .popcnt, .rm, .{ .r16, .rm16 }, .{ 0xf3, 0x0f, 0xb8 }, 0, .short, .popcnt }, - .{ .popcnt, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt }, - .{ .popcnt, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt }, - - // PCLMUL - .{ .pclmulqdq, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .none, .pclmul }, - - // AES - .{ .aesdec, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xde }, 0, .none, .aes }, - - .{ .aesdeclast, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdf }, 0, .none, .aes }, - - .{ .aesenc, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdc }, 0, .none, .aes }, - - .{ .aesenclast, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdd }, 0, .none, .aes }, - - .{ .aesimc, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdb }, 0, .none, .aes }, - - .{ .aeskeygenassist, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xdf }, 0, .none, .aes }, - - // SHA - .{ .sha1rnds4, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0x3a, 0xcc }, 0, .none, .sha }, - - .{ .sha1nexte, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xc8 }, 0, .none, .sha }, - - .{ .sha1msg1, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xc9 }, 0, .none, .sha }, - - .{ .sha1msg2, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xca }, 0, .none, .sha }, - - .{ .sha256rnds2, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xcb }, 0, .none, .sha }, - .{ .sha256rnds2, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x0f, 0x38, 0xcb }, 0, .none, .sha }, - - .{ .sha256msg1, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xcc }, 0, .none, .sha }, - - .{ .sha256msg2, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xcd }, 0, .none, .sha }, - - // AVX - .{ .vaddpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx }, - .{ .vaddpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx }, - - .{ .vaddps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x58 }, 0, .vex_128_wig, .avx }, - .{ .vaddps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x58 }, 0, .vex_256_wig, .avx }, - - .{ .vaddsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx }, - - .{ .vaddss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx }, - - .{ .vaddsubpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd0 }, 0, .vex_128_wig, .avx }, - .{ .vaddsubpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd0 }, 0, .vex_256_wig, .avx }, - - .{ .vaddsubps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .vex_128_wig, .avx }, - .{ .vaddsubps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .vex_256_wig, .avx }, - - .{ .vaesdec, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xde }, 0, .vex_128_wig, .@"aes avx" }, - - .{ .vaesdeclast, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdf }, 0, .vex_128_wig, .@"aes avx" }, - - .{ .vaesenc, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdc }, 0, .vex_128_wig, .@"aes avx" }, - - .{ .vaesenclast, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdd }, 0, .vex_128_wig, .@"aes avx" }, - - .{ .vaesimc, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdb }, 0, .vex_128_wig, .@"aes avx" }, - - .{ .vaeskeygenassist, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xdf }, 0, .vex_128_wig, .@"aes avx" }, - - .{ .vandnpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x55 }, 0, .vex_128_wig, .avx }, - .{ .vandnpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x55 }, 0, .vex_256_wig, .avx }, - - .{ .vandnps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x55 }, 0, .vex_128_wig, .avx }, - .{ .vandnps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x55 }, 0, .vex_256_wig, .avx }, - - .{ .vandpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x54 }, 0, .vex_128_wig, .avx }, - .{ .vandpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x54 }, 0, .vex_256_wig, .avx }, - - .{ .vandps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x54 }, 0, .vex_128_wig, .avx }, - .{ .vandps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x54 }, 0, .vex_256_wig, .avx }, - - .{ .vblendpd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .vex_128_wig, .avx }, - .{ .vblendpd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .vex_256_wig, .avx }, - - .{ .vblendps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .vex_128_wig, .avx }, - .{ .vblendps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .vex_256_wig, .avx }, - - .{ .vblendvpd, .rvmr, .{ .xmm, .xmm, .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x3a, 0x4b }, 0, .vex_128_w0, .avx }, - .{ .vblendvpd, .rvmr, .{ .ymm, .ymm, .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x3a, 0x4b }, 0, .vex_256_w0, .avx }, - - .{ .vblendvps, .rvmr, .{ .xmm, .xmm, .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x3a, 0x4a }, 0, .vex_128_w0, .avx }, - .{ .vblendvps, .rvmr, .{ .ymm, .ymm, .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x3a, 0x4a }, 0, .vex_256_w0, .avx }, - - .{ .vbroadcastss, .rm, .{ .xmm, .m32 }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx }, - .{ .vbroadcastss, .rm, .{ .ymm, .m32 }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx }, - .{ .vbroadcastsd, .rm, .{ .ymm, .m64 }, .{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx }, - .{ .vbroadcastf128, .rm, .{ .ymm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x1a }, 0, .vex_256_w0, .avx }, - - .{ .vcmppd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc2 }, 0, .vex_128_wig, .avx }, - .{ .vcmppd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0xc2 }, 0, .vex_256_wig, .avx }, - - .{ .vcmpps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc2 }, 0, .vex_128_wig, .avx }, - .{ .vcmpps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x0f, 0xc2 }, 0, .vex_256_wig, .avx }, - - .{ .vcmpsd, .rvmi, .{ .xmm, .xmm, .xmm_m64, .imm8 }, .{ 0xf2, 0x0f, 0xc2 }, 0, .vex_lig_wig, .avx }, - - .{ .vcmpss, .rvmi, .{ .xmm, .xmm, .xmm_m32, .imm8 }, .{ 0xf3, 0x0f, 0xc2 }, 0, .vex_lig_wig, .avx }, - - .{ .vcomisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2f }, 0, .vex_lig_wig, .avx }, - - .{ .vcomiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2f }, 0, .vex_lig_wig, .avx }, - - .{ .vcvtdq2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx }, - .{ .vcvtdq2pd, .rm, .{ .ymm, .xmm_m128 }, .{ 0xf3, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx }, - - .{ .vcvtdq2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5b }, 0, .vex_128_wig, .avx }, - .{ .vcvtdq2ps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x5b }, 0, .vex_256_wig, .avx }, - - .{ .vcvtpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx }, - .{ .vcvtpd2dq, .rm, .{ .xmm, .ymm_m256 }, .{ 0xf2, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx }, - - .{ .vcvtpd2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5a }, 0, .vex_128_wig, .avx }, - .{ .vcvtpd2ps, .rm, .{ .xmm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5a }, 0, .vex_256_wig, .avx }, - - .{ .vcvtps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5b }, 0, .vex_128_wig, .avx }, - .{ .vcvtps2dq, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5b }, 0, .vex_256_wig, .avx }, - - .{ .vcvtps2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x0f, 0x5a }, 0, .vex_128_wig, .avx }, - .{ .vcvtps2pd, .rm, .{ .ymm, .xmm_m128 }, .{ 0x0f, 0x5a }, 0, .vex_256_wig, .avx }, - - .{ .vcvtsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .vex_lig_w0, .sse2 }, - .{ .vcvtsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .vex_lig_w1, .sse2 }, - - .{ .vcvtsd2ss, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx }, - - .{ .vcvtsi2sd, .rvm, .{ .xmm, .xmm, .rm32 }, .{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx }, - .{ .vcvtsi2sd, .rvm, .{ .xmm, .xmm, .rm64 }, .{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx }, - - .{ .vcvtsi2ss, .rvm, .{ .xmm, .xmm, .rm32 }, .{ 0xf3, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx }, - .{ .vcvtsi2ss, .rvm, .{ .xmm, .xmm, .rm64 }, .{ 0xf3, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx }, - - .{ .vcvtss2sd, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx }, - - .{ .vcvtss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .vex_lig_w0, .avx }, - .{ .vcvtss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .vex_lig_w1, .avx }, - - .{ .vcvttpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx }, - .{ .vcvttpd2dq, .rm, .{ .xmm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx }, - - .{ .vcvttps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x5b }, 0, .vex_128_wig, .avx }, - .{ .vcvttps2dq, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x5b }, 0, .vex_256_wig, .avx }, - - .{ .vcvttsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .vex_lig_w0, .sse2 }, - .{ .vcvttsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .vex_lig_w1, .sse2 }, - - .{ .vcvttss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .vex_lig_w0, .avx }, - .{ .vcvttss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .vex_lig_w1, .avx }, - - .{ .vdppd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x41 }, 0, .vex_128_wig, .avx }, - - .{ .vdpps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x40 }, 0, .vex_128_wig, .avx }, - .{ .vdpps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x40 }, 0, .vex_256_wig, .avx }, - - .{ .vdivpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5e }, 0, .vex_128_wig, .avx }, - .{ .vdivpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5e }, 0, .vex_256_wig, .avx }, - - .{ .vdivps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5e }, 0, .vex_128_wig, .avx }, - .{ .vdivps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5e }, 0, .vex_256_wig, .avx }, - - .{ .vdivsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx }, - - .{ .vdivss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx }, - - .{ .vextractf128, .mri, .{ .xmm_m128, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x19 }, 0, .vex_256_w0, .avx }, - - .{ .vextractps, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .vex_128_wig, .avx }, - - .{ .vgf2p8affineinvqb, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xcf }, 0, .vex_128_w1, .@"gfni avx" }, - .{ .vgf2p8affineinvqb, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xcf }, 0, .vex_256_w1, .@"gfni avx" }, - - .{ .vgf2p8affineqb, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xce }, 0, .vex_128_w1, .@"gfni avx" }, - .{ .vgf2p8affineqb, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xce }, 0, .vex_256_w1, .@"gfni avx" }, - - .{ .vgf2p8mulb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xcf }, 0, .vex_128_w0, .@"gfni avx" }, - .{ .vgf2p8mulb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xcf }, 0, .vex_256_w0, .@"gfni avx" }, - - .{ .vhaddpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x7c }, 0, .vex_128_wig, .avx }, - .{ .vhaddpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x7c }, 0, .vex_256_wig, .avx }, - - .{ .vhaddps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0x7c }, 0, .vex_128_wig, .avx }, - .{ .vhaddps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0xf2, 0x0f, 0x7c }, 0, .vex_256_wig, .avx }, - - .{ .vinsertf128, .rvmi, .{ .ymm, .ymm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x18 }, 0, .vex_256_w0, .avx }, - - .{ .vinsertps, .rvmi, .{ .xmm, .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .vex_128_wig, .avx }, - - .{ .vlddqu, .rm, .{ .xmm, .m128 }, .{ 0xf2, 0x0f, 0xf0 }, 0, .vex_128_wig, .avx }, - .{ .vlddqu, .rm, .{ .ymm, .m256 }, .{ 0xf2, 0x0f, 0xf0 }, 0, .vex_256_wig, .avx }, - - .{ .vldmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 2, .vex_lz_wig, .avx }, - - .{ .vmaskmovps, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x2c }, 0, .vex_128_w0, .avx }, - .{ .vmaskmovps, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x2c }, 0, .vex_256_w0, .avx }, - .{ .vmaskmovpd, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x2d }, 0, .vex_128_w0, .avx }, - .{ .vmaskmovpd, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x2d }, 0, .vex_256_w0, .avx }, - .{ .vmaskmovps, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x2e }, 0, .vex_128_w0, .avx }, - .{ .vmaskmovps, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x2e }, 0, .vex_256_w0, .avx }, - .{ .vmaskmovpd, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x2f }, 0, .vex_128_w0, .avx }, - .{ .vmaskmovpd, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x2f }, 0, .vex_256_w0, .avx }, - - .{ .vmaxpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5f }, 0, .vex_128_wig, .avx }, - .{ .vmaxpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5f }, 0, .vex_256_wig, .avx }, - - .{ .vmaxps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5f }, 0, .vex_128_wig, .avx }, - .{ .vmaxps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5f }, 0, .vex_256_wig, .avx }, - - .{ .vmaxsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx }, - - .{ .vmaxss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx }, - - .{ .vmovmskps, .rm, .{ .r32, .xmm }, .{ 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, - .{ .vmovmskps, .rm, .{ .r64, .xmm }, .{ 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, - .{ .vmovmskps, .rm, .{ .r32, .ymm }, .{ 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, - .{ .vmovmskps, .rm, .{ .r64, .ymm }, .{ 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, - - .{ .vmovmskpd, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, - .{ .vmovmskpd, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, - .{ .vmovmskpd, .rm, .{ .r32, .ymm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, - .{ .vmovmskpd, .rm, .{ .r64, .ymm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, - - .{ .vminpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5d }, 0, .vex_128_wig, .avx }, - .{ .vminpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5d }, 0, .vex_256_wig, .avx }, - - .{ .vminps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5d }, 0, .vex_128_wig, .avx }, - .{ .vminps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5d }, 0, .vex_256_wig, .avx }, - - .{ .vminsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx }, - - .{ .vminss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx }, - - .{ .vmovapd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x28 }, 0, .vex_128_wig, .avx }, - .{ .vmovapd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x29 }, 0, .vex_128_wig, .avx }, - .{ .vmovapd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x28 }, 0, .vex_256_wig, .avx }, - .{ .vmovapd, .mr, .{ .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x29 }, 0, .vex_256_wig, .avx }, - - .{ .vmovaps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x28 }, 0, .vex_128_wig, .avx }, - .{ .vmovaps, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x29 }, 0, .vex_128_wig, .avx }, - .{ .vmovaps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x28 }, 0, .vex_256_wig, .avx }, - .{ .vmovaps, .mr, .{ .ymm_m256, .ymm }, .{ 0x0f, 0x29 }, 0, .vex_256_wig, .avx }, - - .{ .vmovd, .rm, .{ .xmm, .rm32 }, .{ 0x66, 0x0f, 0x6e }, 0, .vex_128_w0, .avx }, - .{ .vmovq, .rm, .{ .xmm, .rm64 }, .{ 0x66, 0x0f, 0x6e }, 0, .vex_128_w1, .avx }, - .{ .vmovd, .mr, .{ .rm32, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .vex_128_w0, .avx }, - .{ .vmovq, .mr, .{ .rm64, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .vex_128_w1, .avx }, - - .{ .vmovddup, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, - .{ .vmovddup, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx }, - - .{ .vmovdqa, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6f }, 0, .vex_128_wig, .avx }, - .{ .vmovdqa, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x7f }, 0, .vex_128_wig, .avx }, - .{ .vmovdqa, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6f }, 0, .vex_256_wig, .avx }, - .{ .vmovdqa, .mr, .{ .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x7f }, 0, .vex_256_wig, .avx }, - - .{ .vmovdqu, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x6f }, 0, .vex_128_wig, .avx }, - .{ .vmovdqu, .mr, .{ .xmm_m128, .xmm }, .{ 0xf3, 0x0f, 0x7f }, 0, .vex_128_wig, .avx }, - .{ .vmovdqu, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x6f }, 0, .vex_256_wig, .avx }, - .{ .vmovdqu, .mr, .{ .ymm_m256, .ymm }, .{ 0xf3, 0x0f, 0x7f }, 0, .vex_256_wig, .avx }, - - .{ .vmovhlps, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, - - .{ .vmovhpd, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x66, 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, - .{ .vmovhpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x17 }, 0, .vex_128_wig, .avx }, - - .{ .vmovhps, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, - .{ .vmovhps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x17 }, 0, .vex_128_wig, .avx }, - - .{ .vmovlhps, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, - - .{ .vmovlpd, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x66, 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, - .{ .vmovlpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x13 }, 0, .vex_128_wig, .avx }, - - .{ .vmovlps, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, - .{ .vmovlps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x13 }, 0, .vex_128_wig, .avx }, - - .{ .vmovq, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx }, - .{ .vmovq, .mr, .{ .xmm_m64, .xmm }, .{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx }, - - .{ .vmovsd, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, - .{ .vmovsd, .rm, .{ .xmm, .m64 }, .{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, - .{ .vmovsd, .mvr, .{ .xmm, .xmm, .xmm }, .{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, - .{ .vmovsd, .mr, .{ .m64, .xmm }, .{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, - - .{ .vmovshdup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, - .{ .vmovshdup, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x16 }, 0, .vex_256_wig, .avx }, - - .{ .vmovsldup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, - .{ .vmovsldup, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x12 }, 0, .vex_256_wig, .avx }, - - .{ .vmovss, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, - .{ .vmovss, .rm, .{ .xmm, .m32 }, .{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, - .{ .vmovss, .mvr, .{ .xmm, .xmm, .xmm }, .{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, - .{ .vmovss, .mr, .{ .m32, .xmm }, .{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, - - .{ .vmovupd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x10 }, 0, .vex_128_wig, .avx }, - .{ .vmovupd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x11 }, 0, .vex_128_wig, .avx }, - .{ .vmovupd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x10 }, 0, .vex_256_wig, .avx }, - .{ .vmovupd, .mr, .{ .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x11 }, 0, .vex_256_wig, .avx }, - - .{ .vmovups, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x10 }, 0, .vex_128_wig, .avx }, - .{ .vmovups, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x11 }, 0, .vex_128_wig, .avx }, - .{ .vmovups, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x10 }, 0, .vex_256_wig, .avx }, - .{ .vmovups, .mr, .{ .ymm_m256, .ymm }, .{ 0x0f, 0x11 }, 0, .vex_256_wig, .avx }, - - .{ .vmulpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x59 }, 0, .vex_128_wig, .avx }, - .{ .vmulpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x59 }, 0, .vex_256_wig, .avx }, - - .{ .vmulps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x59 }, 0, .vex_128_wig, .avx }, - .{ .vmulps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x59 }, 0, .vex_256_wig, .avx }, - - .{ .vmulsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx }, - - .{ .vmulss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx }, - - .{ .vorpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x56 }, 0, .vex_128_wig, .avx }, - .{ .vorpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x56 }, 0, .vex_256_wig, .avx }, - - .{ .vorps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx }, - .{ .vorps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx }, - - .{ .vpabsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_128_wig, .avx }, - .{ .vpabsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_128_wig, .avx }, - .{ .vpabsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_128_wig, .avx }, - - .{ .vpacksswb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x63 }, 0, .vex_128_wig, .avx }, - .{ .vpackssdw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6b }, 0, .vex_128_wig, .avx }, - - .{ .vpackusdw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_128_wig, .avx }, - - .{ .vpackuswb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x67 }, 0, .vex_128_wig, .avx }, - - .{ .vpaddb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfc }, 0, .vex_128_wig, .avx }, - .{ .vpaddw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfd }, 0, .vex_128_wig, .avx }, - .{ .vpaddd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfe }, 0, .vex_128_wig, .avx }, - .{ .vpaddq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd4 }, 0, .vex_128_wig, .avx }, - - .{ .vpaddsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xec }, 0, .vex_128_wig, .avx }, - .{ .vpaddsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xed }, 0, .vex_128_wig, .avx }, - - .{ .vpaddusb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdc }, 0, .vex_128_wig, .avx }, - .{ .vpaddusw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdd }, 0, .vex_128_wig, .avx }, - - .{ .vpalignr, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .vex_128_wig, .avx }, - - .{ .vpand, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdb }, 0, .vex_128_wig, .avx }, - - .{ .vpandn, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx }, - - .{ .vpblendvb, .rvmr, .{ .xmm, .xmm, .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_128_w0, .avx }, - - .{ .vpblendw, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_128_wig, .avx }, - - .{ .vpclmulqdq, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" }, - - .{ .vpcmpeqb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx }, - .{ .vpcmpeqw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x75 }, 0, .vex_128_wig, .avx }, - .{ .vpcmpeqd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x76 }, 0, .vex_128_wig, .avx }, - - .{ .vpcmpeqq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x29 }, 0, .vex_128_wig, .avx }, - - .{ .vpcmpgtb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x64 }, 0, .vex_128_wig, .avx }, - .{ .vpcmpgtw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x65 }, 0, .vex_128_wig, .avx }, - .{ .vpcmpgtd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x66 }, 0, .vex_128_wig, .avx }, - - .{ .vpcmpgtq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x37 }, 0, .vex_128_wig, .avx }, - - .{ .vperm2f128, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x06 }, 0, .vex_256_w0, .avx }, - - .{ .vpermilpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0d }, 0, .vex_128_w0, .avx }, - .{ .vpermilpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0d }, 0, .vex_256_w0, .avx }, - .{ .vpermilpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x05 }, 0, .vex_128_w0, .avx }, - .{ .vpermilpd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x05 }, 0, .vex_256_w0, .avx }, - - .{ .vpermilpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0c }, 0, .vex_128_w0, .avx }, - .{ .vpermilps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x04 }, 0, .vex_128_w0, .avx }, - .{ .vpermilps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0c }, 0, .vex_256_w0, .avx }, - .{ .vpermilps, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x04 }, 0, .vex_256_w0, .avx }, - - .{ .vpextrb, .mri, .{ .r32_m8, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx }, - .{ .vpextrd, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx }, - .{ .vpextrq, .mri, .{ .rm64, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx }, - - .{ .vpextrw, .rmi, .{ .r32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0xc5 }, 0, .vex_128_w0, .avx }, - .{ .vpextrw, .mri, .{ .r32_m16, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_w0, .avx }, - - .{ .vphaddw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x01 }, 0, .vex_128_wig, .avx }, - .{ .vphaddd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x02 }, 0, .vex_128_wig, .avx }, - - .{ .vphaddsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x03 }, 0, .vex_128_wig, .avx }, - - .{ .vphminposuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x41 }, 0, .vex_128_wig, .avx }, - - .{ .vphsubw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x05 }, 0, .vex_128_wig, .avx }, - .{ .vphsubd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x06 }, 0, .vex_128_wig, .avx }, - - .{ .vphsubsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x07 }, 0, .vex_128_wig, .avx }, - - .{ .vpinsrb, .rvmi, .{ .xmm, .xmm, .r32_m8, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx }, - .{ .vpinsrd, .rvmi, .{ .xmm, .xmm, .rm32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx }, - .{ .vpinsrq, .rvmi, .{ .xmm, .xmm, .rm64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx }, - - .{ .vpinsrw, .rvmi, .{ .xmm, .xmm, .r32_m16, .imm8 }, .{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_w0, .avx }, - - .{ .vpmaddubsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x04 }, 0, .vex_128_wig, .avx }, - - .{ .vpmaddwd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf5 }, 0, .vex_128_wig, .avx }, - - .{ .vpmaxsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx }, - .{ .vpmaxsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx }, - .{ .vpmaxsd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3d }, 0, .vex_128_wig, .avx }, - - .{ .vpmaxub, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xde }, 0, .vex_128_wig, .avx }, - .{ .vpmaxuw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3e }, 0, .vex_128_wig, .avx }, - - .{ .vpmaxud, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3f }, 0, .vex_128_wig, .avx }, - - .{ .vpminsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x38 }, 0, .vex_128_wig, .avx }, - .{ .vpminsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xea }, 0, .vex_128_wig, .avx }, - .{ .vpminsd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x39 }, 0, .vex_128_wig, .avx }, - - .{ .vpminub, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xda }, 0, .vex_128_wig, .avx }, - .{ .vpminuw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3a }, 0, .vex_128_wig, .avx }, - - .{ .vpminud, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3b }, 0, .vex_128_wig, .avx }, - - .{ .vpmovmskb, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx }, - .{ .vpmovmskb, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx }, - - .{ .vpmovsxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_128_wig, .avx }, - .{ .vpmovsxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_128_wig, .avx }, - .{ .vpmovsxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_128_wig, .avx }, - .{ .vpmovsxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_128_wig, .avx }, - .{ .vpmovsxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_128_wig, .avx }, - .{ .vpmovsxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_128_wig, .avx }, - - .{ .vpmovzxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_128_wig, .avx }, - .{ .vpmovzxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_128_wig, .avx }, - .{ .vpmovzxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_128_wig, .avx }, - .{ .vpmovzxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_128_wig, .avx }, - .{ .vpmovzxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx }, - .{ .vpmovzxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx }, - - .{ .vpmuldq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .vex_128_wig, .avx }, - - .{ .vpmulhrsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0b }, 0, .vex_128_wig, .avx }, - - .{ .vpmulhuw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe4 }, 0, .vex_128_wig, .avx }, - - .{ .vpmulhw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx }, - - .{ .vpmulld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx }, - - .{ .vpmullw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd5 }, 0, .vex_128_wig, .avx }, - - .{ .vpmuludq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf4 }, 0, .vex_128_wig, .avx }, - - .{ .vpor, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx }, - - .{ .vpsadbw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf6 }, 0, .vex_128_wig, .avx }, - - .{ .vpshufb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx }, - - .{ .vpshufd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx }, - - .{ .vpshufhw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx }, - - .{ .vpshuflw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf2, 0x0f, 0x70 }, 0, .vex_128_wig, .avx }, - - .{ .vpsignb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x08 }, 0, .vex_128_wig, .avx }, - .{ .vpsignw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x09 }, 0, .vex_128_wig, .avx }, - .{ .vpsignd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0a }, 0, .vex_128_wig, .avx }, - - .{ .vpsllw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf1 }, 0, .vex_128_wig, .avx }, - .{ .vpsllw, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 6, .vex_128_wig, .avx }, - .{ .vpslld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf2 }, 0, .vex_128_wig, .avx }, - .{ .vpslld, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 6, .vex_128_wig, .avx }, - .{ .vpsllq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf3 }, 0, .vex_128_wig, .avx }, - .{ .vpsllq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 6, .vex_128_wig, .avx }, - - .{ .vpslldq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 7, .vex_128_wig, .avx }, - - .{ .vpsraw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe1 }, 0, .vex_128_wig, .avx }, - .{ .vpsraw, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 4, .vex_128_wig, .avx }, - .{ .vpsrad, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe2 }, 0, .vex_128_wig, .avx }, - .{ .vpsrad, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 4, .vex_128_wig, .avx }, - - .{ .vpsrlw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx }, - .{ .vpsrlw, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 2, .vex_128_wig, .avx }, - .{ .vpsrld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd2 }, 0, .vex_128_wig, .avx }, - .{ .vpsrld, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 2, .vex_128_wig, .avx }, - .{ .vpsrlq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd3 }, 0, .vex_128_wig, .avx }, - .{ .vpsrlq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 2, .vex_128_wig, .avx }, - - .{ .vpsrldq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 3, .vex_128_wig, .avx }, - - .{ .vpsubb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf8 }, 0, .vex_128_wig, .avx }, - .{ .vpsubw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf9 }, 0, .vex_128_wig, .avx }, - .{ .vpsubd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfa }, 0, .vex_128_wig, .avx }, - - .{ .vpsubsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe8 }, 0, .vex_128_wig, .avx }, - .{ .vpsubsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe9 }, 0, .vex_128_wig, .avx }, - - .{ .vpsubq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfb }, 0, .vex_128_wig, .avx }, - - .{ .vpsubusb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd8 }, 0, .vex_128_wig, .avx }, - .{ .vpsubusw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd9 }, 0, .vex_128_wig, .avx }, - - .{ .vptest, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_128_wig, .avx }, - .{ .vptest, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_256_wig, .avx }, - - .{ .vpunpckhbw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx }, - .{ .vpunpckhwd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx }, - .{ .vpunpckhdq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx }, - .{ .vpunpckhqdq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6d }, 0, .vex_128_wig, .avx }, - - .{ .vpunpcklbw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x60 }, 0, .vex_128_wig, .avx }, - .{ .vpunpcklwd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x61 }, 0, .vex_128_wig, .avx }, - .{ .vpunpckldq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx }, - .{ .vpunpcklqdq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx }, - - .{ .vpxor, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xef }, 0, .vex_128_wig, .avx }, - - .{ .vroundpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_128_wig, .avx }, - .{ .vroundpd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_256_wig, .avx }, - - .{ .vroundps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_128_wig, .avx }, - .{ .vroundps, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_256_wig, .avx }, - - .{ .vroundsd, .rvmi, .{ .xmm, .xmm, .xmm_m64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .vex_lig_wig, .avx }, - - .{ .vroundss, .rvmi, .{ .xmm, .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .vex_lig_wig, .avx }, - - .{ .vshufpd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc6 }, 0, .vex_128_wig, .avx }, - .{ .vshufpd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0xc6 }, 0, .vex_256_wig, .avx }, - - .{ .vshufps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc6 }, 0, .vex_128_wig, .avx }, - .{ .vshufps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x0f, 0xc6 }, 0, .vex_256_wig, .avx }, - - .{ .vsqrtpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx }, - .{ .vsqrtpd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx }, - - .{ .vsqrtps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x51 }, 0, .vex_128_wig, .avx }, - .{ .vsqrtps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x51 }, 0, .vex_256_wig, .avx }, - - .{ .vsqrtsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx }, - - .{ .vsqrtss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx }, - - .{ .vstmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 3, .vex_lz_wig, .avx }, - - .{ .vsubpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5c }, 0, .vex_128_wig, .avx }, - .{ .vsubpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5c }, 0, .vex_256_wig, .avx }, - - .{ .vsubps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5c }, 0, .vex_128_wig, .avx }, - .{ .vsubps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5c }, 0, .vex_256_wig, .avx }, - - .{ .vsubsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx }, - - .{ .vsubss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx }, - - .{ .vtestps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_128_w0, .avx }, - .{ .vtestps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_256_w0, .avx }, - .{ .vtestpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_128_w0, .avx }, - .{ .vtestpd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_256_w0, .avx }, - - .{ .vucomisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2e }, 0, .vex_lig_wig, .avx }, - - .{ .vucomiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2e }, 0, .vex_lig_wig, .avx }, - - .{ .vunpckhpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx }, - .{ .vunpckhpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x15 }, 0, .vex_256_wig, .avx }, - - .{ .vunpckhps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x15 }, 0, .vex_128_wig, .avx }, - .{ .vunpckhps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x15 }, 0, .vex_256_wig, .avx }, - - .{ .vunpcklpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x14 }, 0, .vex_128_wig, .avx }, - .{ .vunpcklpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x14 }, 0, .vex_256_wig, .avx }, - - .{ .vunpcklps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x14 }, 0, .vex_128_wig, .avx }, - .{ .vunpcklps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x14 }, 0, .vex_256_wig, .avx }, - - .{ .vxorpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx }, - .{ .vxorpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx }, - - .{ .vxorps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x57 }, 0, .vex_128_wig, .avx }, - .{ .vxorps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x57 }, 0, .vex_256_wig, .avx }, - - // BMI - .{ .andn, .rvm, .{ .r32, .r32, .rm32 }, .{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w0, .bmi }, - .{ .andn, .rvm, .{ .r64, .r64, .rm64 }, .{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w1, .bmi }, - - .{ .bextr, .rmv, .{ .r32, .rm32, .r32 }, .{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi }, - .{ .bextr, .rmv, .{ .r64, .rm64, .r64 }, .{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi }, - - .{ .blsi, .vm, .{ .r32, .rm32 }, .{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w0, .bmi }, - .{ .blsi, .vm, .{ .r64, .rm64 }, .{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w1, .bmi }, - - .{ .blsmsk, .vm, .{ .r32, .rm32 }, .{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w0, .bmi }, - .{ .blsmsk, .vm, .{ .r64, .rm64 }, .{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w1, .bmi }, - - .{ .blsr, .vm, .{ .r32, .rm32 }, .{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w0, .bmi }, - .{ .blsr, .vm, .{ .r64, .rm64 }, .{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w1, .bmi }, - - .{ .tzcnt, .rm, .{ .r16, .rm16 }, .{ 0xf3, 0x0f, 0xbc }, 0, .short, .bmi }, - .{ .tzcnt, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi }, - .{ .tzcnt, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi }, - - // BMI2 - .{ .bzhi, .rmv, .{ .r32, .rm32, .r32 }, .{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, - .{ .bzhi, .rmv, .{ .r64, .rm64, .r64 }, .{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, - - .{ .mulx, .rvm, .{ .r32, .r32, .rm32 }, .{ 0xf2, 0x0f, 0x38, 0xf6 }, 0, .vex_lz_w0, .bmi2 }, - .{ .mulx, .rvm, .{ .r64, .r64, .rm64 }, .{ 0xf2, 0x0f, 0x38, 0xf6 }, 0, .vex_lz_w1, .bmi2 }, - - .{ .pdep, .rvm, .{ .r32, .r32, .rm32 }, .{ 0xf2, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, - .{ .pdep, .rvm, .{ .r64, .r64, .rm64 }, .{ 0xf2, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, - - .{ .pext, .rvm, .{ .r32, .r32, .rm32 }, .{ 0xf3, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, - .{ .pext, .rvm, .{ .r64, .r64, .rm64 }, .{ 0xf3, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, - - .{ .rorx, .rmi, .{ .r32, .rm32, .imm8 }, .{ 0xf2, 0x0f, 0x3a, 0xf0 }, 0, .vex_lz_w0, .bmi2 }, - .{ .rorx, .rmi, .{ .r64, .rm64, .imm8 }, .{ 0xf2, 0x0f, 0x3a, 0xf0 }, 0, .vex_lz_w1, .bmi2 }, - - .{ .sarx, .rmv, .{ .r32, .rm32, .r32 }, .{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, - .{ .shlx, .rmv, .{ .r32, .rm32, .r32 }, .{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, - .{ .shrx, .rmv, .{ .r32, .rm32, .r32 }, .{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, - .{ .sarx, .rmv, .{ .r64, .rm64, .r64 }, .{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, - .{ .shlx, .rmv, .{ .r64, .rm64, .r64 }, .{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, - .{ .shrx, .rmv, .{ .r64, .rm64, .r64 }, .{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, - - // F16C - .{ .vcvtph2ps, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c }, - .{ .vcvtph2ps, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c }, - - .{ .vcvtps2ph, .mri, .{ .xmm_m64, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128_w0, .f16c }, - .{ .vcvtps2ph, .mri, .{ .xmm_m128, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_256_w0, .f16c }, - - // FMA - .{ .vfmadd132pd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w1, .fma }, - .{ .vfmadd213pd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w1, .fma }, - .{ .vfmadd231pd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w1, .fma }, - .{ .vfmadd132pd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w1, .fma }, - .{ .vfmadd213pd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w1, .fma }, - .{ .vfmadd231pd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w1, .fma }, - - .{ .vfmadd132ps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w0, .fma }, - .{ .vfmadd213ps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w0, .fma }, - .{ .vfmadd231ps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w0, .fma }, - .{ .vfmadd132ps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w0, .fma }, - .{ .vfmadd213ps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w0, .fma }, - .{ .vfmadd231ps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w0, .fma }, - - .{ .vfmadd132sd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w1, .fma }, - .{ .vfmadd213sd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w1, .fma }, - .{ .vfmadd231sd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w1, .fma }, - - .{ .vfmadd132ss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w0, .fma }, - .{ .vfmadd213ss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w0, .fma }, - .{ .vfmadd231ss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w0, .fma }, - - // VPCLMULQDQ - .{ .vpclmulqdq, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_256_wig, .vpclmulqdq }, - - // AVX2 - .{ .vbroadcastss, .rm, .{ .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx2 }, - .{ .vbroadcastss, .rm, .{ .ymm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 }, - .{ .vbroadcastsd, .rm, .{ .ymm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 }, - - .{ .vextracti128, .mri, .{ .xmm_m128, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 }, - - .{ .vinserti128, .rvmi, .{ .ymm, .ymm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 }, - - .{ .vpabsb, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 }, - .{ .vpabsd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 }, - .{ .vpabsw, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 }, - - .{ .vpacksswb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x63 }, 0, .vex_256_wig, .avx2 }, - .{ .vpackssdw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6b }, 0, .vex_256_wig, .avx2 }, - - .{ .vpackusdw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_256_wig, .avx2 }, - - .{ .vpackuswb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x67 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpaddb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfc }, 0, .vex_256_wig, .avx2 }, - .{ .vpaddw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfd }, 0, .vex_256_wig, .avx2 }, - .{ .vpaddd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfe }, 0, .vex_256_wig, .avx2 }, - .{ .vpaddq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd4 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpaddsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xec }, 0, .vex_256_wig, .avx2 }, - .{ .vpaddsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xed }, 0, .vex_256_wig, .avx2 }, - - .{ .vpaddusb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdc }, 0, .vex_256_wig, .avx2 }, - .{ .vpaddusw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdd }, 0, .vex_256_wig, .avx2 }, - - .{ .vpalignr, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .vex_256_wig, .avx2 }, - - .{ .vpand, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdb }, 0, .vex_256_wig, .avx2 }, - - .{ .vpandn, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 }, - - .{ .vpblendd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_128_w0, .avx2 }, - .{ .vpblendd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_256_w0, .avx2 }, - - .{ .vpblendvb, .rvmr, .{ .ymm, .ymm, .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_256_w0, .avx2 }, - - .{ .vpblendw, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_256_wig, .avx2 }, - - .{ .vpbroadcastb, .rm, .{ .xmm, .xmm_m8 }, .{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 }, - .{ .vpbroadcastb, .rm, .{ .ymm, .xmm_m8 }, .{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 }, - .{ .vpbroadcastw, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 }, - .{ .vpbroadcastw, .rm, .{ .ymm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_256_w0, .avx2 }, - .{ .vpbroadcastd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_128_w0, .avx2 }, - .{ .vpbroadcastd, .rm, .{ .ymm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 }, - .{ .vpbroadcastq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 }, - .{ .vpbroadcastq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 }, - .{ .vbroadcasti128, .rm, .{ .ymm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 }, - - .{ .vpcmpeqb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 }, - .{ .vpcmpeqw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 }, - .{ .vpcmpeqd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpcmpeqq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x29 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpcmpgtb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x64 }, 0, .vex_256_wig, .avx2 }, - .{ .vpcmpgtw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x65 }, 0, .vex_256_wig, .avx2 }, - .{ .vpcmpgtd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x66 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpcmpgtq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x37 }, 0, .vex_256_wig, .avx2 }, - - .{ .vphaddw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x01 }, 0, .vex_256_wig, .avx2 }, - .{ .vphaddd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x02 }, 0, .vex_256_wig, .avx2 }, - - .{ .vphaddsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x03 }, 0, .vex_256_wig, .avx2 }, - - .{ .vphaddw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x05 }, 0, .vex_256_wig, .avx2 }, - .{ .vphaddd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x06 }, 0, .vex_256_wig, .avx2 }, - - .{ .vphaddsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x07 }, 0, .vex_256_wig, .avx2 }, - - .{ .vperm2i128, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x46 }, 0, .vex_256_w0, .avx2 }, - - .{ .vpermd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x36 }, 0, .vex_256_w0, .avx2 }, - - .{ .vpermpd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x01 }, 0, .vex_256_w1, .avx2 }, - - .{ .vpermps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x16 }, 0, .vex_256_w0, .avx2 }, - - .{ .vpermq, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x00 }, 0, .vex_256_w1, .avx2 }, - - .{ .vpmaddubsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x04 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmaddwd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf5 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmaskmovd, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_128_w0, .avx2 }, - .{ .vpmaskmovd, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_256_w0, .avx2 }, - .{ .vpmaskmovq, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_128_w1, .avx2 }, - .{ .vpmaskmovq, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_256_w1, .avx2 }, - .{ .vpmaskmovd, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_128_w0, .avx2 }, - .{ .vpmaskmovd, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_256_w0, .avx2 }, - .{ .vpmaskmovq, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_128_w1, .avx2 }, - .{ .vpmaskmovq, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_256_w1, .avx2 }, - - .{ .vpmaxsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_256_wig, .avx2 }, - .{ .vpmaxsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xee }, 0, .vex_256_wig, .avx2 }, - .{ .vpmaxsd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3d }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmaxub, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xde }, 0, .vex_256_wig, .avx2 }, - .{ .vpmaxuw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3e }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmaxud, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3f }, 0, .vex_256_wig, .avx2 }, - - .{ .vpminsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x38 }, 0, .vex_256_wig, .avx2 }, - .{ .vpminsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xea }, 0, .vex_256_wig, .avx2 }, - .{ .vpminsd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x39 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpminub, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xda }, 0, .vex_256_wig, .avx2 }, - .{ .vpminuw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3a }, 0, .vex_256_wig, .avx2 }, - - .{ .vpminud, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3b }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmovmskb, .rm, .{ .r32, .ymm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovmskb, .rm, .{ .r64, .ymm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmovsxbw, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovsxbd, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovsxbq, .rm, .{ .ymm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovsxwd, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovsxwq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovsxdq, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmovzxbw, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovzxbd, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovzxbq, .rm, .{ .ymm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovzxwd, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovzxwq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 }, - .{ .vpmovzxdq, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmuldq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmulhrsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0b }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmulhuw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe4 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmulhw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmulld, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmullw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd5 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpmuludq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf4 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpor, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 }, - - .{ .vpsadbw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf6 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpshufb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 }, - .{ .vpshufd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpshufhw, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpshuflw, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0xf2, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpsignb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x08 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsignw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x09 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsignd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0a }, 0, .vex_256_wig, .avx2 }, - - .{ .vpsllw, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf1 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsllw, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 6, .vex_256_wig, .avx2 }, - .{ .vpslld, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf2 }, 0, .vex_256_wig, .avx2 }, - .{ .vpslld, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 6, .vex_256_wig, .avx2 }, - .{ .vpsllq, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf3 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsllq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 6, .vex_256_wig, .avx2 }, - - .{ .vpslldq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 7, .vex_256_wig, .avx2 }, - - .{ .vpsllvd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_128_w0, .avx2 }, - .{ .vpsllvq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_128_w1, .avx2 }, - .{ .vpsllvd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_256_w0, .avx2 }, - .{ .vpsllvq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_256_w1, .avx2 }, - - .{ .vpsraw, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe1 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsraw, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 4, .vex_256_wig, .avx2 }, - .{ .vpsrad, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe2 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsrad, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 4, .vex_256_wig, .avx2 }, - - .{ .vpsravd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x46 }, 0, .vex_128_w0, .avx2 }, - .{ .vpsravd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x46 }, 0, .vex_256_w0, .avx2 }, - - .{ .vpsrlw, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd1 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsrlw, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 2, .vex_256_wig, .avx2 }, - .{ .vpsrld, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd2 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsrld, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 2, .vex_256_wig, .avx2 }, - .{ .vpsrlq, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd3 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsrlq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 2, .vex_256_wig, .avx2 }, - - .{ .vpsrldq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 3, .vex_256_wig, .avx2 }, - - .{ .vpsrlvd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_128_w0, .avx2 }, - .{ .vpsrlvq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_128_w1, .avx2 }, - .{ .vpsrlvd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_256_w0, .avx2 }, - .{ .vpsrlvq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_256_w1, .avx2 }, - - .{ .vpsubb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf8 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsubw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf9 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsubd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfa }, 0, .vex_256_wig, .avx2 }, - - .{ .vpsubsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe8 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsubsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe9 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpsubq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfb }, 0, .vex_256_wig, .avx2 }, - - .{ .vpsubusb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd8 }, 0, .vex_256_wig, .avx2 }, - .{ .vpsubusw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd9 }, 0, .vex_256_wig, .avx2 }, - - .{ .vpunpckhbw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x68 }, 0, .vex_256_wig, .avx2 }, - .{ .vpunpckhwd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x69 }, 0, .vex_256_wig, .avx2 }, - .{ .vpunpckhdq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6a }, 0, .vex_256_wig, .avx2 }, - .{ .vpunpckhqdq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6d }, 0, .vex_256_wig, .avx2 }, - - .{ .vpunpcklbw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x60 }, 0, .vex_256_wig, .avx2 }, - .{ .vpunpcklwd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x61 }, 0, .vex_256_wig, .avx2 }, - .{ .vpunpckldq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x62 }, 0, .vex_256_wig, .avx2 }, - .{ .vpunpcklqdq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6c }, 0, .vex_256_wig, .avx2 }, - - .{ .vpxor, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xef }, 0, .vex_256_wig, .avx2 }, - - // ADX - .{ .adcx, .rm, .{ .r32, .rm32 }, .{ 0x66, 0x0f, 0x38, 0xf6 }, 0, .none, .adx }, - .{ .adcx, .rm, .{ .r64, .rm64 }, .{ 0x66, 0x0f, 0x38, 0xf6 }, 0, .long, .adx }, - - .{ .adox, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0x38, 0xf6 }, 0, .none, .adx }, - .{ .adox, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0x38, 0xf6 }, 0, .long, .adx }, - - // VAES - .{ .vaesdec, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xde }, 0, .vex_256_wig, .vaes }, - - .{ .vaesdeclast, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xdf }, 0, .vex_256_wig, .vaes }, - - .{ .vaesenc, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xdc }, 0, .vex_256_wig, .vaes }, - - .{ .vaesenclast, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xdd }, 0, .vex_256_wig, .vaes }, - - // AESKLE - .{ .aesdec128kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xdd }, 0, .none, .kl }, - - .{ .aesdec256kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xdf }, 0, .none, .kl }, - - .{ .aesenc128kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xdc }, 0, .none, .kl }, - - .{ .aesenc256kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xde }, 0, .none, .kl }, - - .{ .encodekey128, .rm, .{ .r32, .r32 }, .{ 0xf3, 0x0f, 0x38, 0xfa }, 0, .none, .kl }, - - .{ .encodekey256, .rm, .{ .r32, .r32 }, .{ 0xf3, 0x0f, 0x38, 0xfb }, 0, .none, .kl }, - - .{ .loadiwkey, .rm, .{ .xmm, .xmm }, .{ 0xf3, 0x0f, 0x38, 0xdc }, 0, .none, .kl }, - .{ .loadiwkey, .rm, .{ .xmm, .xmm, .eax, .xmm0 }, .{ 0xf3, 0x0f, 0x38, 0xdc }, 0, .none, .kl }, - - // AESKLEWIDE_KL - .{ .aesdecwide128kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 1, .none, .widekl }, - - .{ .aesdecwide256kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 3, .none, .widekl }, - - .{ .aesencwide128kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 0, .none, .widekl }, - - .{ .aesencwide256kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 2, .none, .widekl }, -} -// zig fmt: on diff --git a/src/codegen.zig b/src/codegen.zig index 4cbf3f56169ec9227f961a72720898cc23df3f96..ed046e966cd54ccf9cde708222ca66aadbc198c4 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -55,11 +55,11 @@ fn importBackend(comptime backend: std.builtin.CompilerBackend) type { .stage2_c => @import("codegen/c.zig"), .stage2_llvm => @import("codegen/llvm.zig"), .stage2_powerpc => unreachable, - .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"), - .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"), + .stage2_riscv64 => @import("codegen/riscv64/CodeGen.zig"), + .stage2_sparc64 => @import("codegen/sparc64/CodeGen.zig"), .stage2_spirv => @import("codegen/spirv/CodeGen.zig"), - .stage2_wasm => @import("arch/wasm/CodeGen.zig"), - .stage2_x86, .stage2_x86_64 => @import("arch/x86_64/CodeGen.zig"), + .stage2_wasm => @import("codegen/wasm/CodeGen.zig"), + .stage2_x86, .stage2_x86_64 => @import("codegen/x86_64/CodeGen.zig"), _ => unreachable, }; } @@ -99,10 +99,10 @@ pub fn wantsLiveness(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) bool { /// union of all MIR types. The active tag is known from the backend in use; see `AnyMir.tag`. pub const AnyMir = union { aarch64: if (dev.env.supports(.aarch64_backend)) @import("codegen/aarch64/Mir.zig") else noreturn, - riscv64: if (dev.env.supports(.riscv64_backend)) @import("arch/riscv64/Mir.zig") else noreturn, - sparc64: if (dev.env.supports(.sparc64_backend)) @import("arch/sparc64/Mir.zig") else noreturn, - x86_64: if (dev.env.supports(.x86_64_backend)) @import("arch/x86_64/Mir.zig") else noreturn, - wasm: if (dev.env.supports(.wasm_backend)) @import("arch/wasm/Mir.zig") else noreturn, + riscv64: if (dev.env.supports(.riscv64_backend)) @import("codegen/riscv64/Mir.zig") else noreturn, + sparc64: if (dev.env.supports(.sparc64_backend)) @import("codegen/sparc64/Mir.zig") else noreturn, + x86_64: if (dev.env.supports(.x86_64_backend)) @import("codegen/x86_64/Mir.zig") else noreturn, + wasm: if (dev.env.supports(.wasm_backend)) @import("codegen/wasm/Mir.zig") else noreturn, c: if (dev.env.supports(.c_backend)) @import("codegen/c.zig").Mir else noreturn, pub inline fn tag(comptime backend: std.builtin.CompilerBackend) []const u8 { diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index fd9972d86746161099a7fd95b490f5edfbf51109..04c231c5ec07c191cd1462c841ade3736dce689e 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -21,11 +21,11 @@ const Air = @import("../Air.zig"); const Value = @import("../Value.zig"); const Type = @import("../Type.zig"); const codegen = @import("../codegen.zig"); -const x86_64_abi = @import("../arch/x86_64/abi.zig"); +const x86_64_abi = @import("x86_64/abi.zig"); const wasm_c_abi = @import("wasm/abi.zig"); const aarch64_c_abi = @import("aarch64/abi.zig"); const arm_c_abi = @import("arm/abi.zig"); -const riscv_c_abi = @import("../arch/riscv64/abi.zig"); +const riscv_c_abi = @import("riscv64/abi.zig"); const mips_c_abi = @import("mips/abi.zig"); const dev = @import("../dev.zig"); diff --git a/src/codegen/riscv64/CodeGen.zig b/src/codegen/riscv64/CodeGen.zig new file mode 100644 index 0000000000000000000000000000000000000000..fe40ba4bbb16ffcbcc2768af9a6f8a57600c1e4f --- /dev/null +++ b/src/codegen/riscv64/CodeGen.zig @@ -0,0 +1,8494 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const build_options = @import("build_options"); + +const mem = std.mem; +const math = std.math; +const assert = std.debug.assert; +const Allocator = mem.Allocator; + +const Air = @import("../../Air.zig"); +const Mir = @import("Mir.zig"); +const Emit = @import("Emit.zig"); +const Type = @import("../../Type.zig"); +const Value = @import("../../Value.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); +const Package = @import("../../Package.zig"); +const InternPool = @import("../../InternPool.zig"); +const Compilation = @import("../../Compilation.zig"); +const target_util = @import("../../target.zig"); +const trace = @import("../../tracy.zig").trace; +const codegen = @import("../../codegen.zig"); + +const ErrorMsg = Zcu.ErrorMsg; +const Target = std.Target; + +const log = std.log.scoped(.riscv_codegen); +const tracking_log = std.log.scoped(.tracking); +const verbose_tracking_log = std.log.scoped(.verbose_tracking); +const wip_mir_log = std.log.scoped(.wip_mir); +const Alignment = InternPool.Alignment; + +const CodeGenError = codegen.CodeGenError; + +const bits = @import("bits.zig"); +const abi = @import("abi.zig"); +const Lower = @import("Lower.zig"); +const mnem_import = @import("mnem.zig"); +const Mnemonic = mnem_import.Mnemonic; +const Pseudo = mnem_import.Pseudo; +const encoding = @import("encoding.zig"); + +const Register = bits.Register; +const CSR = bits.CSR; +const Immediate = bits.Immediate; +const Memory = bits.Memory; +const FrameIndex = bits.FrameIndex; +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; +const Instruction = encoding.Instruction; + +const InnerError = CodeGenError || error{OutOfRegisters}; + +pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { + return comptime &.initMany(&.{ + .expand_intcast_safe, + .expand_int_from_float_safe, + .expand_int_from_float_optimized_safe, + .expand_add_safe, + .expand_sub_safe, + .expand_mul_safe, + }); +} + +pt: Zcu.PerThread, +air: Air, +liveness: Air.Liveness, +bin_file: *link.File, +gpa: Allocator, + +mod: *Package.Module, +target: *const std.Target, +args: []MCValue, +ret_mcv: InstTracking, +func_index: InternPool.Index, +fn_type: Type, +arg_index: usize, +src_loc: Zcu.LazySrcLoc, + +mir_instructions: std.MultiArrayList(Mir.Inst) = .{}, + +owner: Owner, + +/// Byte offset within the source file of the ending curly. +end_di_line: u32, +end_di_column: u32, + +scope_generation: u32, + +/// The value is an offset into the `Function` `code` from the beginning. +/// To perform the reloc, write 32-bit signed little-endian integer +/// which is a relative jump, based on the address following the reloc. +exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, + +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, + +/// Whenever there is a runtime branch, we push a Branch onto this stack, +/// and pop it off when the runtime branch joins. This provides an "overlay" +/// of the table of mappings from instructions to `MCValue` from within the branch. +/// This way we can modify the `MCValue` for an instruction in different ways +/// within different branches. Special consideration is needed when a branch +/// joins with its parent, to make sure all instructions have the same MCValue +/// across each runtime branch upon joining. +branch_stack: *std.array_list.Managed(Branch), + +// Currently set vector properties, null means they haven't been set yet in the function. +avl: ?u64, +vtype: ?bits.VType, + +// Key is the block instruction +blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, +register_manager: RegisterManager = .{}, + +const_tracking: ConstTrackingMap = .{}, +inst_tracking: InstTrackingMap = .{}, + +frame_allocs: std.MultiArrayList(FrameAlloc) = .{}, +free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty, +frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{}, + +loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { + /// The state to restore before branching. + state: State, + /// The branch target. + jmp_target: Mir.Inst.Index, +}) = .{}, + +/// Debug field, used to find bugs in the compiler. +air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init, + +const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; + +const SymbolOffset = struct { sym: u32, off: i32 = 0 }; +const RegisterOffset = struct { reg: Register, off: i32 = 0 }; +pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 }; + +const Owner = union(enum) { + nav_index: InternPool.Nav.Index, + lazy_sym: link.File.LazySymbol, + + fn getSymbolIndex(owner: Owner, func: *Func) !u32 { + const pt = func.pt; + switch (owner) { + .nav_index => |nav_index| { + const elf_file = func.bin_file.cast(.elf).?; + return elf_file.zigObjectPtr().?.getOrCreateMetadataForNav(pt.zcu, nav_index); + }, + .lazy_sym => |lazy_sym| { + const elf_file = func.bin_file.cast(.elf).?; + return elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err| + func.fail("{s} creating lazy symbol", .{@errorName(err)}); + }, + } + } +}; + +const MCValue = union(enum) { + /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. + /// TODO Look into deleting this tag and using `dead` instead, since every use + /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. + none, + /// Control flow will not allow this value to be observed. + unreach, + /// No more references to this value remain. + /// The payload is the value of scope_generation at the point where the death occurred + dead: u32, + /// The value is undefined. Contains a symbol index to an undefined constant. Null means + /// set the undefined value via immediate instead of a load. + undef: ?u32, + /// A pointer-sized integer that fits in a register. + /// If the type is a pointer, this is the pointer address in virtual address space. + immediate: u64, + /// The value doesn't exist in memory yet. + load_symbol: SymbolOffset, + /// The address of the memory location not-yet-allocated by the linker. + lea_symbol: SymbolOffset, + /// The value is in a target-specific register. + register: Register, + /// The value is split across two registers + register_pair: [2]Register, + /// The value is in memory at a hard-coded address. + /// If the type is a pointer, it means the pointer address is at this memory location. + memory: u64, + /// The value stored at an offset from a frame index + /// Payload is a frame address. + load_frame: FrameAddr, + /// The address of an offset from a frame index + /// Payload is a frame address. + lea_frame: FrameAddr, + air_ref: Air.Inst.Ref, + /// The value is in memory at a constant offset from the address in a register. + indirect: RegisterOffset, + /// The value is a constant offset from the value in a register. + register_offset: RegisterOffset, + /// This indicates that we have already allocated a frame index for this instruction, + /// but it has not been spilled there yet in the current control flow. + /// Payload is a frame index. + reserved_frame: FrameIndex, + + fn isMemory(mcv: MCValue) bool { + return switch (mcv) { + .memory, .indirect, .load_frame => true, + else => false, + }; + } + + fn isImmediate(mcv: MCValue) bool { + return switch (mcv) { + .immediate => true, + else => false, + }; + } + + fn isRegister(mcv: MCValue) bool { + return switch (mcv) { + .register => true, + .register_offset => |reg_off| return reg_off.off == 0, + else => false, + }; + } + + fn isMutable(mcv: MCValue) bool { + return switch (mcv) { + .none => unreachable, + .unreach => unreachable, + .dead => unreachable, + + .immediate, + .memory, + .lea_frame, + .undef, + .lea_symbol, + .air_ref, + .reserved_frame, + => false, + + .register, + .register_pair, + .register_offset, + .load_symbol, + .indirect, + => true, + + .load_frame => |frame_addr| !frame_addr.index.isNamed(), + }; + } + + fn address(mcv: MCValue) MCValue { + return switch (mcv) { + .none, + .unreach, + .dead, + .immediate, + .lea_frame, + .register_offset, + .register_pair, + .register, + .undef, + .air_ref, + .lea_symbol, + .reserved_frame, + => unreachable, // not in memory + + .load_symbol => |sym_off| .{ .lea_symbol = sym_off }, + .memory => |addr| .{ .immediate = addr }, + .load_frame => |off| .{ .lea_frame = off }, + .indirect => |reg_off| switch (reg_off.off) { + 0 => .{ .register = reg_off.reg }, + else => .{ .register_offset = reg_off }, + }, + }; + } + + fn deref(mcv: MCValue) MCValue { + return switch (mcv) { + .none, + .unreach, + .dead, + .memory, + .indirect, + .undef, + .air_ref, + .register_pair, + .load_frame, + .load_symbol, + .reserved_frame, + => unreachable, // not a pointer + + .immediate => |addr| .{ .memory = addr }, + .register => |reg| .{ .indirect = .{ .reg = reg } }, + .register_offset => |reg_off| .{ .indirect = reg_off }, + .lea_frame => |off| .{ .load_frame = off }, + .lea_symbol => |sym_off| .{ .load_symbol = sym_off }, + }; + } + + fn offset(mcv: MCValue, off: i32) MCValue { + return switch (mcv) { + .none, + .unreach, + .dead, + .undef, + .air_ref, + .reserved_frame, + => unreachable, // not valid + .register_pair, + .memory, + .indirect, + .load_symbol, + .lea_symbol, + => switch (off) { + 0 => mcv, + else => unreachable, + }, + .load_frame => |frame| .{ .load_frame = .{ .index = frame.index, .off = frame.off + off } }, + .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) }, + .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } }, + .register_offset => |reg_off| .{ .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off } }, + .lea_frame => |frame_addr| .{ + .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, + }, + }; + } + + fn getReg(mcv: MCValue) ?Register { + return switch (mcv) { + .register => |reg| reg, + .register_offset, .indirect => |ro| ro.reg, + else => null, + }; + } + + fn getRegs(mcv: *const MCValue) []const Register { + return switch (mcv.*) { + .register => |*reg| @as(*const [1]Register, reg), + .register_pair => |*regs| regs, + .register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg), + else => &.{}, + }; + } +}; + +const Branch = struct { + inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty, + + fn deinit(func: *Branch, gpa: Allocator) void { + func.inst_table.deinit(gpa); + func.* = undefined; + } +}; + +const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking); +const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking); + +const InstTracking = struct { + long: MCValue, + short: MCValue, + + fn init(result: MCValue) InstTracking { + return .{ .long = switch (result) { + .none, + .unreach, + .undef, + .immediate, + .memory, + .load_frame, + .lea_frame, + .load_symbol, + .lea_symbol, + => result, + .dead, + .reserved_frame, + .air_ref, + => unreachable, + .register, + .register_pair, + .register_offset, + .indirect, + => .none, + }, .short = result }; + } + + fn getReg(inst_tracking: InstTracking) ?Register { + return inst_tracking.short.getReg(); + } + + fn getRegs(inst_tracking: *const InstTracking) []const Register { + return inst_tracking.short.getRegs(); + } + + fn spill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void { + if (std.meta.eql(inst_tracking.long, inst_tracking.short)) return; // Already spilled + // Allocate or reuse frame index + switch (inst_tracking.long) { + .none => inst_tracking.long = try function.allocRegOrMem( + function.typeOfIndex(inst), + inst, + false, + ), + .load_frame => {}, + .reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } }, + else => unreachable, + } + tracking_log.debug("spill %{d} from {} to {}", .{ inst, inst_tracking.short, inst_tracking.long }); + try function.genCopy(function.typeOfIndex(inst), inst_tracking.long, inst_tracking.short); + } + + fn reuseFrame(inst_tracking: *InstTracking) void { + switch (inst_tracking.long) { + .reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } }, + else => {}, + } + inst_tracking.short = switch (inst_tracking.long) { + .none, + .unreach, + .undef, + .immediate, + .memory, + .load_frame, + .lea_frame, + .load_symbol, + .lea_symbol, + => inst_tracking.long, + .dead, + .register, + .register_pair, + .register_offset, + .indirect, + .reserved_frame, + .air_ref, + => unreachable, + }; + } + + fn trackSpill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void { + try function.freeValue(inst_tracking.short); + inst_tracking.reuseFrame(); + tracking_log.debug("%{d} => {f} (spilled)", .{ inst, inst_tracking.* }); + } + + fn verifyMaterialize(inst_tracking: InstTracking, target: InstTracking) void { + switch (inst_tracking.long) { + .none, + .unreach, + .undef, + .immediate, + .memory, + .lea_frame, + .load_symbol, + .lea_symbol, + => assert(std.meta.eql(inst_tracking.long, target.long)), + .load_frame, + .reserved_frame, + => switch (target.long) { + .none, + .load_frame, + .reserved_frame, + => {}, + else => unreachable, + }, + .dead, + .register, + .register_pair, + .register_offset, + .indirect, + .air_ref, + => unreachable, + } + } + + fn materialize( + inst_tracking: *InstTracking, + function: *Func, + inst: Air.Inst.Index, + target: InstTracking, + ) !void { + inst_tracking.verifyMaterialize(target); + try inst_tracking.materializeUnsafe(function, inst, target); + } + + fn materializeUnsafe( + inst_tracking: InstTracking, + function: *Func, + inst: Air.Inst.Index, + target: InstTracking, + ) !void { + const ty = function.typeOfIndex(inst); + if ((inst_tracking.long == .none or inst_tracking.long == .reserved_frame) and target.long == .load_frame) + try function.genCopy(ty, target.long, inst_tracking.short); + try function.genCopy(ty, target.short, inst_tracking.short); + } + + fn trackMaterialize(inst_tracking: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void { + inst_tracking.verifyMaterialize(target); + // Don't clobber reserved frame indices + inst_tracking.long = if (target.long == .none) switch (inst_tracking.long) { + .load_frame => |addr| .{ .reserved_frame = addr.index }, + .reserved_frame => inst_tracking.long, + else => target.long, + } else target.long; + inst_tracking.short = target.short; + tracking_log.debug("%{d} => {f} (materialize)", .{ inst, inst_tracking.* }); + } + + fn resurrect(inst_tracking: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void { + switch (inst_tracking.short) { + .dead => |die_generation| if (die_generation >= scope_generation) { + inst_tracking.reuseFrame(); + tracking_log.debug("%{d} => {f} (resurrect)", .{ inst, inst_tracking.* }); + }, + else => {}, + } + } + + fn die(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void { + if (inst_tracking.short == .dead) return; + try function.freeValue(inst_tracking.short); + inst_tracking.short = .{ .dead = function.scope_generation }; + tracking_log.debug("%{d} => {f} (death)", .{ inst, inst_tracking.* }); + } + + fn reuse( + inst_tracking: *InstTracking, + function: *Func, + new_inst: ?Air.Inst.Index, + old_inst: Air.Inst.Index, + ) void { + inst_tracking.short = .{ .dead = function.scope_generation }; + if (new_inst) |inst| + tracking_log.debug("%{d} => {f} (reuse %{d})", .{ inst, inst_tracking.*, old_inst }) + else + tracking_log.debug("tmp => {f} (reuse %{d})", .{ inst_tracking.*, old_inst }); + } + + fn liveOut(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) void { + for (inst_tracking.getRegs()) |reg| { + if (function.register_manager.isRegFree(reg)) { + tracking_log.debug("%{d} => {f} (live-out)", .{ inst, inst_tracking.* }); + continue; + } + + const index = RegisterManager.indexOfRegIntoTracked(reg).?; + const tracked_inst = function.register_manager.registers[index]; + const tracking = function.getResolvedInstValue(tracked_inst); + + // Disable death. + var found_reg = false; + var remaining_reg: Register = .zero; + for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) { + assert(!found_reg); + found_reg = true; + } else { + assert(remaining_reg == .zero); + remaining_reg = tracked_reg; + }; + assert(found_reg); + tracking.short = switch (remaining_reg) { + .zero => .{ .dead = function.scope_generation }, + else => .{ .register = remaining_reg }, + }; + + // Perform side-effects of freeValue manually. + function.register_manager.freeReg(reg); + + tracking_log.debug("%{d} => {f} (live-out %{d})", .{ inst, inst_tracking.*, tracked_inst }); + } + } + + pub fn format(inst_tracking: InstTracking, writer: *std.Io.Writer) std.Io.Writer.Error!void { + if (!std.meta.eql(inst_tracking.long, inst_tracking.short)) try writer.print("|{}| ", .{inst_tracking.long}); + try writer.print("{}", .{inst_tracking.short}); + } +}; + +const FrameAlloc = struct { + abi_size: u31, + spill_pad: u3, + abi_align: Alignment, + ref_count: u16, + + fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc { + return .{ + .abi_size = @intCast(alloc_abi.size), + .spill_pad = alloc_abi.pad, + .abi_align = alloc_abi.alignment, + .ref_count = 0, + }; + } + fn initType(ty: Type, zcu: *Zcu) FrameAlloc { + return init(.{ + .size = ty.abiSize(zcu), + .alignment = ty.abiAlignment(zcu), + }); + } + fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc { + const abi_size = ty.abiSize(zcu); + const spill_size = if (abi_size < 8) + math.ceilPowerOfTwoAssert(u64, abi_size) + else + std.mem.alignForward(u64, abi_size, 8); + return init(.{ + .size = spill_size, + .pad = @intCast(spill_size - abi_size), + .alignment = ty.abiAlignment(zcu).maxStrict( + Alignment.fromNonzeroByteUnits(@min(spill_size, 8)), + ), + }); + } +}; + +const BlockData = struct { + relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + state: State, + + fn deinit(bd: *BlockData, gpa: Allocator) void { + bd.relocs.deinit(gpa); + bd.* = undefined; + } +}; + +const State = struct { + registers: RegisterManager.TrackedRegisters, + reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, + free_registers: RegisterManager.RegisterBitSet, + inst_tracking_len: u32, + scope_generation: u32, +}; + +fn initRetroactiveState(func: *Func) State { + var state: State = undefined; + state.inst_tracking_len = @intCast(func.inst_tracking.count()); + state.scope_generation = func.scope_generation; + return state; +} + +fn saveRetroactiveState(func: *Func, state: *State) !void { + const free_registers = func.register_manager.free_registers; + var it = free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = func.register_manager.registers[index]; + state.registers[index] = tracked_inst; + state.reg_tracking[index] = func.inst_tracking.get(tracked_inst).?; + } + state.free_registers = free_registers; +} + +fn saveState(func: *Func) !State { + var state = func.initRetroactiveState(); + try func.saveRetroactiveState(&state); + return state; +} + +fn restoreState(func: *Func, state: State, deaths: []const Air.Inst.Index, comptime opts: struct { + emit_instructions: bool, + update_tracking: bool, + resurrect: bool, + close_scope: bool, +}) !void { + if (opts.close_scope) { + for ( + func.inst_tracking.keys()[state.inst_tracking_len..], + func.inst_tracking.values()[state.inst_tracking_len..], + ) |inst, *tracking| try tracking.die(func, inst); + func.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len); + } + + if (opts.resurrect) for ( + func.inst_tracking.keys()[0..state.inst_tracking_len], + func.inst_tracking.values()[0..state.inst_tracking_len], + ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation); + for (deaths) |death| try func.processDeath(death); + + const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + if (opts.update_tracking) {} else std.heap.stackFallback(@sizeOf(ExpectedContents), func.gpa); + + var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity( + stack.get(), + @typeInfo(ExpectedContents).array.len, + ); + defer if (!opts.update_tracking) { + for (reg_locks.items) |lock| func.register_manager.unlockReg(lock); + reg_locks.deinit(); + }; + + for (0..state.registers.len) |index| { + const current_maybe_inst = if (func.register_manager.free_registers.isSet(index)) + null + else + func.register_manager.registers[index]; + const target_maybe_inst = if (state.free_registers.isSet(index)) + null + else + state.registers[index]; + if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst| + assert(func.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len); + if (opts.emit_instructions) { + if (current_maybe_inst) |current_inst| { + try func.inst_tracking.getPtr(current_inst).?.spill(func, current_inst); + } + if (target_maybe_inst) |target_inst| { + const target_tracking = func.inst_tracking.getPtr(target_inst).?; + try target_tracking.materialize(func, target_inst, state.reg_tracking[index]); + } + } + if (opts.update_tracking) { + if (current_maybe_inst) |current_inst| { + try func.inst_tracking.getPtr(current_inst).?.trackSpill(func, current_inst); + } + blk: { + const inst = target_maybe_inst orelse break :blk; + const reg = RegisterManager.regAtTrackedIndex(@intCast(index)); + func.register_manager.freeReg(reg); + func.register_manager.getRegAssumeFree(reg, inst); + } + if (target_maybe_inst) |target_inst| { + func.inst_tracking.getPtr(target_inst).?.trackMaterialize( + target_inst, + state.reg_tracking[index], + ); + } + } else if (target_maybe_inst) |_| + try reg_locks.append(func.register_manager.lockRegIndexAssumeUnused(@intCast(index))); + } + + if (opts.update_tracking and std.debug.runtime_safety) { + assert(func.register_manager.free_registers.eql(state.free_registers)); + var used_reg_it = state.free_registers.iterator(.{ .kind = .unset }); + while (used_reg_it.next()) |index| + assert(func.register_manager.registers[index] == state.registers[index]); + } +} + +const Func = @This(); + +const CallView = enum(u1) { + callee, + caller, +}; + +pub fn generate( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + air: *const Air, + liveness: *const ?Air.Liveness, +) CodeGenError!Mir { + const zcu = pt.zcu; + const gpa = zcu.gpa; + const ip = &zcu.intern_pool; + const func = zcu.funcInfo(func_index); + const fn_type = Type.fromInterned(func.ty); + const mod = zcu.navFileScope(func.owner_nav).mod.?; + + var branch_stack = std.array_list.Managed(Branch).init(gpa); + defer { + assert(branch_stack.items.len == 1); + branch_stack.items[0].deinit(gpa); + branch_stack.deinit(); + } + try branch_stack.append(.{}); + + var function: Func = .{ + .gpa = gpa, + .air = air.*, + .pt = pt, + .mod = mod, + .bin_file = bin_file, + .liveness = liveness.*.?, + .target = &mod.resolved_target.result, + .owner = .{ .nav_index = func.owner_nav }, + .args = undefined, // populated after `resolveCallingConventionValues` + .ret_mcv = undefined, // populated after `resolveCallingConventionValues` + .func_index = func_index, + .fn_type = fn_type, + .arg_index = 0, + .branch_stack = &branch_stack, + .src_loc = src_loc, + .end_di_line = func.rbrace_line, + .end_di_column = func.rbrace_column, + .scope_generation = 0, + .avl = null, + .vtype = null, + }; + defer { + function.frame_allocs.deinit(gpa); + function.free_frame_indices.deinit(gpa); + function.frame_locs.deinit(gpa); + function.loops.deinit(gpa); + var block_it = function.blocks.valueIterator(); + while (block_it.next()) |block| block.deinit(gpa); + function.blocks.deinit(gpa); + function.inst_tracking.deinit(gpa); + function.const_tracking.deinit(gpa); + function.exitlude_jump_relocs.deinit(gpa); + function.mir_instructions.deinit(gpa); + } + + wip_mir_log.debug("{f}:", .{fmtNav(func.owner_nav, ip)}); + + try function.frame_allocs.resize(gpa, FrameIndex.named_count); + function.frame_allocs.set( + @intFromEnum(FrameIndex.stack_frame), + FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }), + ); + function.frame_allocs.set( + @intFromEnum(FrameIndex.call_frame), + FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }), + ); + + const fn_info = zcu.typeToFunc(fn_type).?; + var call_info = function.resolveCallingConventionValues(fn_info, &.{}) catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + else => |e| return e, + }; + + defer call_info.deinit(&function); + + function.args = call_info.args; + function.ret_mcv = call_info.return_value; + function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{ + .size = Type.u64.abiSize(zcu), + .alignment = Type.u64.abiAlignment(zcu).min(call_info.stack_align), + })); + function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{ + .size = Type.u64.abiSize(zcu), + .alignment = Alignment.min( + call_info.stack_align, + Alignment.fromNonzeroByteUnits(function.target.stackAlignment()), + ), + })); + function.frame_allocs.set(@intFromEnum(FrameIndex.args_frame), FrameAlloc.init(.{ + .size = call_info.stack_byte_count, + .alignment = call_info.stack_align, + })); + function.frame_allocs.set(@intFromEnum(FrameIndex.spill_frame), FrameAlloc.init(.{ + .size = 0, + .alignment = Type.u64.abiAlignment(zcu), + })); + + function.gen() catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), + else => |e| return e, + }; + + var mir: Mir = .{ + .instructions = function.mir_instructions.toOwnedSlice(), + .frame_locs = function.frame_locs.toOwnedSlice(), + }; + errdefer mir.deinit(gpa); + return mir; +} + +pub fn generateLazy( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + lazy_sym: link.File.LazySymbol, + atom_index: u32, + w: *std.Io.Writer, + debug_output: link.File.DebugInfoOutput, +) (CodeGenError || std.Io.Writer.Error)!void { + _ = atom_index; + const comp = bin_file.comp; + const gpa = comp.gpa; + const mod = comp.root_mod; + + var function: Func = .{ + .gpa = gpa, + .air = undefined, + .pt = pt, + .mod = mod, + .bin_file = bin_file, + .liveness = undefined, + .target = &mod.resolved_target.result, + .owner = .{ .lazy_sym = lazy_sym }, + .args = undefined, // populated after `resolveCallingConventionValues` + .ret_mcv = undefined, // populated after `resolveCallingConventionValues` + .func_index = undefined, + .fn_type = undefined, + .arg_index = 0, + .branch_stack = undefined, + .src_loc = src_loc, + .end_di_line = undefined, + .end_di_column = undefined, + .scope_generation = 0, + .avl = null, + .vtype = null, + }; + defer function.mir_instructions.deinit(gpa); + + function.genLazy(lazy_sym) catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), + else => |e| return e, + }; + + var mir: Mir = .{ + .instructions = function.mir_instructions.toOwnedSlice(), + .frame_locs = function.frame_locs.toOwnedSlice(), + }; + defer mir.deinit(gpa); + + var emit: Emit = .{ + .lower = .{ + .pt = pt, + .allocator = gpa, + .mir = mir, + .cc = .auto, + .src_loc = src_loc, + .output_mode = comp.config.output_mode, + .link_mode = comp.config.link_mode, + .pic = mod.pic, + }, + .bin_file = bin_file, + .debug_output = debug_output, + .w = w, + .prev_di_pc = undefined, // no debug info yet + .prev_di_line = undefined, // no debug info yet + .prev_di_column = undefined, // no debug info yet + }; + defer emit.deinit(); + + emit.emitMir() catch |err| switch (err) { + error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?), + error.InvalidInstruction => |e| return function.fail("emit MIR failed: {s} (Zig compiler bug)", .{@errorName(e)}), + else => |e| return e, + }; +} + +const FormatWipMirData = struct { + func: *Func, + inst: Mir.Inst.Index, +}; +fn formatWipMir(data: FormatWipMirData, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const pt = data.func.pt; + const comp = pt.zcu.comp; + var lower: Lower = .{ + .pt = pt, + .allocator = data.func.gpa, + .mir = .{ + .instructions = data.func.mir_instructions.slice(), + .frame_locs = data.func.frame_locs.slice(), + }, + .cc = .auto, + .src_loc = data.func.src_loc, + .output_mode = comp.config.output_mode, + .link_mode = comp.config.link_mode, + .pic = comp.root_mod.pic, + }; + var first = true; + for ((lower.lowerMir(data.inst, .{ .allow_frame_locs = false }) catch |err| switch (err) { + error.LowerFail => { + defer { + lower.err_msg.?.deinit(data.func.gpa); + lower.err_msg = null; + } + try writer.writeAll(lower.err_msg.?.msg); + return; + }, + error.OutOfMemory, error.InvalidInstruction => |e| { + try writer.writeAll(switch (e) { + error.OutOfMemory => "Out of memory", + error.InvalidInstruction => "CodeGen failed to find a viable instruction.", + }); + return; + }, + else => |e| return e, + }).insts) |lowered_inst| { + if (!first) try writer.writeAll("\ndebug(wip_mir): "); + try writer.print(" | {}", .{lowered_inst}); + first = false; + } +} +fn fmtWipMir(func: *Func, inst: Mir.Inst.Index) std.fmt.Alt(FormatWipMirData, formatWipMir) { + return .{ .data = .{ .func = func, .inst = inst } }; +} + +const FormatNavData = struct { + ip: *const InternPool, + nav_index: InternPool.Nav.Index, +}; +fn formatNav(data: FormatNavData, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{f}", .{data.ip.getNav(data.nav_index).fqn.fmt(data.ip)}); +} +fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Alt(FormatNavData, formatNav) { + return .{ .data = .{ + .ip = ip, + .nav_index = nav_index, + } }; +} + +const FormatAirData = struct { + func: *Func, + inst: Air.Inst.Index, +}; +fn formatAir(data: FormatAirData, writer: *std.Io.Writer) std.Io.Writer.Error!void { + // Not acceptable implementation because it ignores `writer`: + //data.func.air.dumpInst(data.inst, data.func.pt, data.func.liveness); + _ = data; + _ = writer; + @panic("unimplemented"); +} +fn fmtAir(func: *Func, inst: Air.Inst.Index) std.fmt.Alt(FormatAirData, formatAir) { + return .{ .data = .{ .func = func, .inst = inst } }; +} + +const FormatTrackingData = struct { + func: *Func, +}; +fn formatTracking(data: FormatTrackingData, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var it = data.func.inst_tracking.iterator(); + while (it.next()) |entry| try writer.print("\n%{d} = {f}", .{ entry.key_ptr.*, entry.value_ptr.* }); +} +fn fmtTracking(func: *Func) std.fmt.Alt(FormatTrackingData, formatTracking) { + return .{ .data = .{ .func = func } }; +} + +fn addInst(func: *Func, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { + const gpa = func.gpa; + try func.mir_instructions.ensureUnusedCapacity(gpa, 1); + const result_index: Mir.Inst.Index = @intCast(func.mir_instructions.len); + func.mir_instructions.appendAssumeCapacity(inst); + if (switch (inst.tag) { + else => true, + .pseudo_dbg_prologue_end, + .pseudo_dbg_line_column, + .pseudo_dbg_epilogue_begin, + .pseudo_dead, + => false, + }) wip_mir_log.debug("{f}", .{func.fmtWipMir(result_index)}); + return result_index; +} + +fn addPseudo(func: *Func, mnem: Mnemonic) error{OutOfMemory}!Mir.Inst.Index { + return func.addInst(.{ + .tag = mnem, + .data = .none, + }); +} + +/// Returns a temporary register that contains the value of the `reg` csr. +/// +/// Caller's duty to lock the return register is needed. +fn getCsr(func: *Func, csr: CSR) !Register { + assert(func.hasFeature(.zicsr)); + const dst_reg = try func.register_manager.allocReg(null, func.regTempClassForType(Type.u64)); + _ = try func.addInst(.{ + .tag = .csrrs, + .data = .{ .csr = .{ + .csr = csr, + .rd = dst_reg, + .rs1 = .x0, + } }, + }); + return dst_reg; +} + +fn setVl(func: *Func, dst_reg: Register, avl: u64, options: bits.VType) !void { + if (func.avl == avl) if (func.vtype) |vtype| { + // it's already set, we don't need to do anything + if (@as(u8, @bitCast(vtype)) == @as(u8, @bitCast(options))) return; + }; + + func.avl = avl; + func.vtype = options; + + if (avl == 0) { + // the caller means to do "vsetvli zero, zero ..." which keeps the avl to whatever it was before + const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); + _ = try func.addInst(.{ + .tag = .vsetvli, + .data = .{ .i_type = .{ + .rd = dst_reg, + .rs1 = .zero, + .imm12 = Immediate.u(options_int), + } }, + }); + } else { + // if the avl can fit into u5 we can use vsetivli otherwise use vsetvli + if (avl <= std.math.maxInt(u5)) { + const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options)); + _ = try func.addInst(.{ + .tag = .vsetivli, + .data = .{ + .i_type = .{ + .rd = dst_reg, + .rs1 = @enumFromInt(avl), + .imm12 = Immediate.u(options_int), + }, + }, + }); + } else { + const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); + const temp_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = avl }); + _ = try func.addInst(.{ + .tag = .vsetvli, + .data = .{ .i_type = .{ + .rd = dst_reg, + .rs1 = temp_reg, + .imm12 = Immediate.u(options_int), + } }, + }); + } + } +} + +const required_features = [_]Target.riscv.Feature{ + .d, + .m, + .a, + .zicsr, + .v, + .zbb, +}; + +fn gen(func: *Func) !void { + const pt = func.pt; + const zcu = pt.zcu; + const fn_info = zcu.typeToFunc(func.fn_type).?; + + inline for (required_features) |feature| { + if (!func.hasFeature(feature)) { + return func.fail( + "target missing required feature {s}", + .{@tagName(feature)}, + ); + } + } + + if (fn_info.cc != .naked) { + _ = try func.addPseudo(.pseudo_dbg_prologue_end); + + const backpatch_stack_alloc = try func.addPseudo(.pseudo_dead); + const backpatch_ra_spill = try func.addPseudo(.pseudo_dead); + const backpatch_fp_spill = try func.addPseudo(.pseudo_dead); + const backpatch_fp_add = try func.addPseudo(.pseudo_dead); + const backpatch_spill_callee_preserved_regs = try func.addPseudo(.pseudo_dead); + + switch (func.ret_mcv.long) { + .none, .unreach => {}, + .indirect => { + // The address where to store the return value for the caller is in a + // register which the callee is free to clobber. Therefore, we purposely + // spill it to stack immediately. + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(Type.u64, zcu)); + try func.genSetMem( + .{ .frame = frame_index }, + 0, + Type.u64, + func.ret_mcv.long.address().offset(-func.ret_mcv.short.indirect.off), + ); + func.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } }; + tracking_log.debug("spill {} to {}", .{ func.ret_mcv.long, frame_index }); + }, + else => unreachable, + } + + try func.genBody(func.air.getMainBody()); + + for (func.exitlude_jump_relocs.items) |jmp_reloc| { + func.mir_instructions.items(.data)[jmp_reloc].j_type.inst = + @intCast(func.mir_instructions.len); + } + + _ = try func.addPseudo(.pseudo_dbg_epilogue_begin); + + const backpatch_restore_callee_preserved_regs = try func.addPseudo(.pseudo_dead); + const backpatch_ra_restore = try func.addPseudo(.pseudo_dead); + const backpatch_fp_restore = try func.addPseudo(.pseudo_dead); + const backpatch_stack_alloc_restore = try func.addPseudo(.pseudo_dead); + + // ret + _ = try func.addInst(.{ + .tag = .jalr, + .data = .{ .i_type = .{ + .rd = .zero, + .rs1 = .ra, + .imm12 = Immediate.s(0), + } }, + }); + + const frame_layout = try func.computeFrameLayout(); + const need_save_reg = frame_layout.save_reg_list.count() > 0; + + func.mir_instructions.set(backpatch_stack_alloc, .{ + .tag = .addi, + .data = .{ .i_type = .{ + .rd = .sp, + .rs1 = .sp, + .imm12 = Immediate.s(-@as(i32, @intCast(frame_layout.stack_adjust))), + } }, + }); + func.mir_instructions.set(backpatch_ra_spill, .{ + .tag = .pseudo_store_rm, + .data = .{ .rm = .{ + .r = .ra, + .m = .{ + .base = .{ .frame = .ret_addr }, + .mod = .{ .size = .dword, .unsigned = false }, + }, + } }, + }); + func.mir_instructions.set(backpatch_ra_restore, .{ + .tag = .pseudo_load_rm, + .data = .{ .rm = .{ + .r = .ra, + .m = .{ + .base = .{ .frame = .ret_addr }, + .mod = .{ .size = .dword, .unsigned = false }, + }, + } }, + }); + func.mir_instructions.set(backpatch_fp_spill, .{ + .tag = .pseudo_store_rm, + .data = .{ .rm = .{ + .r = .s0, + .m = .{ + .base = .{ .frame = .base_ptr }, + .mod = .{ .size = .dword, .unsigned = false }, + }, + } }, + }); + func.mir_instructions.set(backpatch_fp_restore, .{ + .tag = .pseudo_load_rm, + .data = .{ .rm = .{ + .r = .s0, + .m = .{ + .base = .{ .frame = .base_ptr }, + .mod = .{ .size = .dword, .unsigned = false }, + }, + } }, + }); + func.mir_instructions.set(backpatch_fp_add, .{ + .tag = .addi, + .data = .{ .i_type = .{ + .rd = .s0, + .rs1 = .sp, + .imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)), + } }, + }); + func.mir_instructions.set(backpatch_stack_alloc_restore, .{ + .tag = .addi, + .data = .{ .i_type = .{ + .rd = .sp, + .rs1 = .sp, + .imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)), + } }, + }); + + if (need_save_reg) { + func.mir_instructions.set(backpatch_spill_callee_preserved_regs, .{ + .tag = .pseudo_spill_regs, + .data = .{ .reg_list = frame_layout.save_reg_list }, + }); + + func.mir_instructions.set(backpatch_restore_callee_preserved_regs, .{ + .tag = .pseudo_restore_regs, + .data = .{ .reg_list = frame_layout.save_reg_list }, + }); + } + } else { + _ = try func.addPseudo(.pseudo_dbg_prologue_end); + try func.genBody(func.air.getMainBody()); + _ = try func.addPseudo(.pseudo_dbg_epilogue_begin); + } + + // Drop them off at the rbrace. + _ = try func.addInst(.{ + .tag = .pseudo_dbg_line_column, + .data = .{ .pseudo_dbg_line_column = .{ + .line = func.end_di_line, + .column = func.end_di_column, + } }, + }); +} + +fn genLazy(func: *Func, lazy_sym: link.File.LazySymbol) InnerError!void { + const pt = func.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) { + .@"enum" => { + const enum_ty = Type.fromInterned(lazy_sym.ty); + wip_mir_log.debug("{f}.@tagName:", .{enum_ty.fmt(pt)}); + + const param_regs = abi.Registers.Integer.function_arg_regs; + const ret_reg = param_regs[0]; + const enum_mcv: MCValue = .{ .register = param_regs[1] }; + + const exitlude_jump_relocs = try func.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu)); + defer func.gpa.free(exitlude_jump_relocs); + + const data_reg, const data_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(data_lock); + + const elf_file = func.bin_file.cast(.elf).?; + const zo = elf_file.zigObjectPtr().?; + const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, .{ + .kind = .const_data, + .ty = enum_ty.toIntern(), + }) catch |err| + return func.fail("{s} creating lazy symbol", .{@errorName(err)}); + + try func.genSetReg(Type.u64, data_reg, .{ .lea_symbol = .{ .sym = sym_index } }); + + const cmp_reg, const cmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(cmp_lock); + + var data_off: i32 = 0; + const tag_names = enum_ty.enumFields(zcu); + for (exitlude_jump_relocs, 0..) |*exitlude_jump_reloc, tag_index| { + const tag_name_len = tag_names.get(ip)[tag_index].length(ip); + const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index)); + const tag_mcv = try func.genTypedValue(tag_val); + + _ = try func.genBinOp( + .cmp_neq, + enum_mcv, + enum_ty, + tag_mcv, + enum_ty, + cmp_reg, + ); + const skip_reloc = try func.condBr(Type.bool, .{ .register = cmp_reg }); + + try func.genSetMem( + .{ .reg = ret_reg }, + 0, + Type.u64, + .{ .register_offset = .{ .reg = data_reg, .off = data_off } }, + ); + + try func.genSetMem( + .{ .reg = ret_reg }, + 8, + Type.u64, + .{ .immediate = tag_name_len }, + ); + + exitlude_jump_reloc.* = try func.addInst(.{ + .tag = .pseudo_j, + .data = .{ .j_type = .{ + .rd = .zero, + .inst = undefined, + } }, + }); + func.performReloc(skip_reloc); + + data_off += @intCast(tag_name_len + 1); + } + + try func.airTrap(); + + for (exitlude_jump_relocs) |reloc| func.performReloc(reloc); + + _ = try func.addInst(.{ + .tag = .jalr, + .data = .{ .i_type = .{ + .rd = .zero, + .rs1 = .ra, + .imm12 = Immediate.s(0), + } }, + }); + }, + else => return func.fail( + "TODO implement {s} for {f}", + .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, + ), + } +} + +fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { + const pt = func.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const air_tags = func.air.instructions.items(.tag); + + for (body) |inst| { + if (func.liveness.isUnused(inst) and !func.air.mustLower(inst, ip)) continue; + wip_mir_log.debug("{f}", .{func.fmtAir(inst)}); + verbose_tracking_log.debug("{f}", .{func.fmtTracking()}); + + const old_air_bookkeeping = func.air_bookkeeping; + try func.ensureProcessDeathCapacity(Air.Liveness.bpi); + + func.reused_operands = @TypeOf(func.reused_operands).initEmpty(); + try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1); + const tag = air_tags[@intFromEnum(inst)]; + switch (tag) { + // zig fmt: off + .add, + .add_wrap, + .sub, + .sub_wrap, + + .add_sat, + + .mul, + .mul_wrap, + .div_trunc, + .div_exact, + .rem, + + .shl, .shl_exact, + .shr, .shr_exact, + + .bool_and, + .bool_or, + .bit_and, + .bit_or, + + .xor, + + .min, + .max, + => try func.airBinOp(inst, tag), + + + .ptr_add, + .ptr_sub => try func.airPtrArithmetic(inst, tag), + + .mod, + .div_float, + .div_floor, + => return func.fail("TODO: {s}", .{@tagName(tag)}), + + .sqrt, + .sin, + .cos, + .tan, + .exp, + .exp2, + .log, + .log2, + .log10, + .floor, + .ceil, + .round, + .trunc_float, + .neg, + => try func.airUnaryMath(inst, tag), + + .add_with_overflow => try func.airAddWithOverflow(inst), + .sub_with_overflow => try func.airSubWithOverflow(inst), + .mul_with_overflow => try func.airMulWithOverflow(inst), + .shl_with_overflow => try func.airShlWithOverflow(inst), + + + .sub_sat => try func.airSubSat(inst), + .mul_sat => try func.airMulSat(inst), + .shl_sat => try func.airShlSat(inst), + + .add_safe, + .sub_safe, + .mul_safe, + .intcast_safe, + .int_from_float_safe, + .int_from_float_optimized_safe, + => return func.fail("TODO implement safety_checked_instructions", .{}), + + .cmp_lt, + .cmp_lte, + .cmp_eq, + .cmp_gte, + .cmp_gt, + .cmp_neq, + => try func.airCmp(inst, tag), + + .cmp_vector => try func.airCmpVector(inst), + .cmp_lt_errors_len => try func.airCmpLtErrorsLen(inst), + + .slice => try func.airSlice(inst), + .array_to_slice => try func.airArrayToSlice(inst), + + .slice_ptr => try func.airSlicePtr(inst), + .slice_len => try func.airSliceLen(inst), + + .alloc => try func.airAlloc(inst), + .ret_ptr => try func.airRetPtr(inst), + .arg => try func.airArg(inst), + .assembly => try func.airAsm(inst), + .bitcast => try func.airBitCast(inst), + .block => try func.airBlock(inst), + .br => try func.airBr(inst), + .repeat => try func.airRepeat(inst), + .switch_dispatch => try func.airSwitchDispatch(inst), + .trap => try func.airTrap(), + .breakpoint => try func.airBreakpoint(), + .ret_addr => try func.airRetAddr(inst), + .frame_addr => try func.airFrameAddress(inst), + .cond_br => try func.airCondBr(inst), + .dbg_stmt => try func.airDbgStmt(inst), + .dbg_empty_stmt => func.finishAirBookkeeping(), + .fptrunc => try func.airFptrunc(inst), + .fpext => try func.airFpext(inst), + .intcast => try func.airIntCast(inst), + .trunc => try func.airTrunc(inst), + .is_non_null => try func.airIsNonNull(inst), + .is_non_null_ptr => try func.airIsNonNullPtr(inst), + .is_null => try func.airIsNull(inst), + .is_null_ptr => try func.airIsNullPtr(inst), + .is_non_err => try func.airIsNonErr(inst), + .is_non_err_ptr => try func.airIsNonErrPtr(inst), + .is_err => try func.airIsErr(inst), + .is_err_ptr => try func.airIsErrPtr(inst), + .load => try func.airLoad(inst), + .loop => try func.airLoop(inst), + .not => try func.airNot(inst), + .ret => try func.airRet(inst, false), + .ret_safe => try func.airRet(inst, true), + .ret_load => try func.airRetLoad(inst), + .store => try func.airStore(inst, false), + .store_safe => try func.airStore(inst, true), + .struct_field_ptr=> try func.airStructFieldPtr(inst), + .struct_field_val=> try func.airStructFieldVal(inst), + .float_from_int => try func.airFloatFromInt(inst), + .int_from_float => try func.airIntFromFloat(inst), + .cmpxchg_strong => try func.airCmpxchg(inst, .strong), + .cmpxchg_weak => try func.airCmpxchg(inst, .weak), + .atomic_rmw => try func.airAtomicRmw(inst), + .atomic_load => try func.airAtomicLoad(inst), + .memcpy => try func.airMemcpy(inst), + .memmove => try func.airMemmove(inst), + .memset => try func.airMemset(inst, false), + .memset_safe => try func.airMemset(inst, true), + .set_union_tag => try func.airSetUnionTag(inst), + .get_union_tag => try func.airGetUnionTag(inst), + .clz => try func.airClz(inst), + .ctz => try func.airCtz(inst), + .popcount => try func.airPopcount(inst), + .abs => try func.airAbs(inst), + .byte_swap => try func.airByteSwap(inst), + .bit_reverse => try func.airBitReverse(inst), + .tag_name => try func.airTagName(inst), + .error_name => try func.airErrorName(inst), + .splat => try func.airSplat(inst), + .select => try func.airSelect(inst), + .shuffle_one => try func.airShuffleOne(inst), + .shuffle_two => try func.airShuffleTwo(inst), + .reduce => try func.airReduce(inst), + .aggregate_init => try func.airAggregateInit(inst), + .union_init => try func.airUnionInit(inst), + .prefetch => try func.airPrefetch(inst), + .mul_add => try func.airMulAdd(inst), + .addrspace_cast => return func.fail("TODO: addrspace_cast", .{}), + + .@"try" => try func.airTry(inst), + .try_cold => try func.airTry(inst), + .try_ptr => return func.fail("TODO: try_ptr", .{}), + .try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}), + + .dbg_var_ptr, + .dbg_var_val, + .dbg_arg_inline, + => try func.airDbgVar(inst), + + .dbg_inline_block => try func.airDbgInlineBlock(inst), + + .call => try func.airCall(inst, .auto), + .call_always_tail => try func.airCall(inst, .always_tail), + .call_never_tail => try func.airCall(inst, .never_tail), + .call_never_inline => try func.airCall(inst, .never_inline), + + .atomic_store_unordered => try func.airAtomicStore(inst, .unordered), + .atomic_store_monotonic => try func.airAtomicStore(inst, .monotonic), + .atomic_store_release => try func.airAtomicStore(inst, .release), + .atomic_store_seq_cst => try func.airAtomicStore(inst, .seq_cst), + .struct_field_ptr_index_0 => try func.airStructFieldPtrIndex(inst, 0), + .struct_field_ptr_index_1 => try func.airStructFieldPtrIndex(inst, 1), + .struct_field_ptr_index_2 => try func.airStructFieldPtrIndex(inst, 2), + .struct_field_ptr_index_3 => try func.airStructFieldPtrIndex(inst, 3), + + .field_parent_ptr => try func.airFieldParentPtr(inst), + + .switch_br => try func.airSwitchBr(inst), + .loop_switch_br => try func.airLoopSwitchBr(inst), + + .ptr_slice_len_ptr => try func.airPtrSliceLenPtr(inst), + .ptr_slice_ptr_ptr => try func.airPtrSlicePtrPtr(inst), + + .array_elem_val => try func.airArrayElemVal(inst), + + .slice_elem_val => try func.airSliceElemVal(inst), + .slice_elem_ptr => try func.airSliceElemPtr(inst), + + .ptr_elem_val => try func.airPtrElemVal(inst), + .ptr_elem_ptr => try func.airPtrElemPtr(inst), + + .inferred_alloc, .inferred_alloc_comptime => unreachable, + .unreach => func.finishAirBookkeeping(), + + .optional_payload => try func.airOptionalPayload(inst), + .optional_payload_ptr => try func.airOptionalPayloadPtr(inst), + .optional_payload_ptr_set => try func.airOptionalPayloadPtrSet(inst), + .unwrap_errunion_err => try func.airUnwrapErrErr(inst), + .unwrap_errunion_payload => try func.airUnwrapErrPayload(inst), + .unwrap_errunion_err_ptr => try func.airUnwrapErrErrPtr(inst), + .unwrap_errunion_payload_ptr=> try func.airUnwrapErrPayloadPtr(inst), + .errunion_payload_ptr_set => try func.airErrUnionPayloadPtrSet(inst), + .err_return_trace => try func.airErrReturnTrace(inst), + .set_err_return_trace => try func.airSetErrReturnTrace(inst), + .save_err_return_trace_index=> try func.airSaveErrReturnTraceIndex(inst), + + .wrap_optional => try func.airWrapOptional(inst), + .wrap_errunion_payload => try func.airWrapErrUnionPayload(inst), + .wrap_errunion_err => try func.airWrapErrUnionErr(inst), + + .runtime_nav_ptr => try func.airRuntimeNavPtr(inst), + + .add_optimized, + .sub_optimized, + .mul_optimized, + .div_float_optimized, + .div_trunc_optimized, + .div_floor_optimized, + .div_exact_optimized, + .rem_optimized, + .mod_optimized, + .neg_optimized, + .cmp_lt_optimized, + .cmp_lte_optimized, + .cmp_eq_optimized, + .cmp_gte_optimized, + .cmp_gt_optimized, + .cmp_neq_optimized, + .cmp_vector_optimized, + .reduce_optimized, + .int_from_float_optimized, + => return func.fail("TODO implement optimized float mode", .{}), + + .is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}), + .error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}), + .vector_store_elem => return func.fail("TODO implement vector_store_elem", .{}), + + .c_va_arg => return func.fail("TODO implement c_va_arg", .{}), + .c_va_copy => return func.fail("TODO implement c_va_copy", .{}), + .c_va_end => return func.fail("TODO implement c_va_end", .{}), + .c_va_start => return func.fail("TODO implement c_va_start", .{}), + + .wasm_memory_size => unreachable, + .wasm_memory_grow => unreachable, + + .work_item_id => unreachable, + .work_group_size => unreachable, + .work_group_id => unreachable, + // zig fmt: on + } + + assert(!func.register_manager.lockedRegsExist()); + + if (std.debug.runtime_safety) { + if (func.air_bookkeeping < old_air_bookkeeping + 1) { + std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] }); + } + + { // check consistency of tracked registers + var it = func.register_manager.free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = func.register_manager.registers[index]; + tracking_log.debug("tracked inst: {f}", .{tracked_inst}); + const tracking = func.getResolvedInstValue(tracked_inst); + for (tracking.getRegs()) |reg| { + if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; + } else return std.debug.panic( + \\%{f} takes up these regs: {any}, however this regs {any}, don't use it + , .{ tracked_inst, tracking.getRegs(), RegisterManager.regAtTrackedIndex(@intCast(index)) }); + } + } + } + } + verbose_tracking_log.debug("{f}", .{func.fmtTracking()}); +} + +fn getValue(func: *Func, value: MCValue, inst: ?Air.Inst.Index) !void { + for (value.getRegs()) |reg| try func.register_manager.getReg(reg, inst); +} + +fn getValueIfFree(func: *Func, value: MCValue, inst: ?Air.Inst.Index) void { + for (value.getRegs()) |reg| if (func.register_manager.isRegFree(reg)) + func.register_manager.getRegAssumeFree(reg, inst); +} + +fn freeValue(func: *Func, value: MCValue) !void { + switch (value) { + .register => |reg| func.register_manager.freeReg(reg), + .register_pair => |regs| for (regs) |reg| func.register_manager.freeReg(reg), + .register_offset => |reg_off| func.register_manager.freeReg(reg_off.reg), + else => {}, // TODO process stack allocation death + } +} + +fn feed(func: *Func, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void { + if (bt.feed()) if (operand.toIndex()) |inst| { + log.debug("feed inst: %{f}", .{inst}); + try func.processDeath(inst); + }; +} + +/// Asserts there is already capacity to insert into top branch inst_table. +fn processDeath(func: *Func, inst: Air.Inst.Index) !void { + try func.inst_tracking.getPtr(inst).?.die(func, inst); +} + +/// Called when there are no operands, and the instruction is always unreferenced. +fn finishAirBookkeeping(func: *Func) void { + if (std.debug.runtime_safety) { + func.air_bookkeeping += 1; + } +} + +fn finishAirResult(func: *Func, inst: Air.Inst.Index, result: MCValue) void { + if (func.liveness.isUnused(inst)) switch (result) { + .none, .dead, .unreach => {}, + // Why didn't the result die? + .register => |r| if (r != .zero) unreachable, + else => unreachable, + } else { + switch (result) { + .register => |r| if (r == .zero) unreachable, // Why did we discard a used result? + else => {}, + } + + tracking_log.debug("%{d} => {} (birth)", .{ inst, result }); + func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result)); + // In some cases, an operand may be reused as the result. + // If that operand died and was a register, it was freed by + // processDeath, so we have to "re-allocate" the register. + func.getValueIfFree(result, inst); + } + func.finishAirBookkeeping(); +} + +fn finishAir( + func: *Func, + inst: Air.Inst.Index, + result: MCValue, + operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, +) !void { + const tomb_bits = func.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (func.reused_operands.isSet(op_index)) continue; + try func.processDeath(op.toIndexAllowNone() orelse continue); + } + func.finishAirResult(inst, result); +} + +const FrameLayout = struct { + stack_adjust: i12, + save_reg_list: Mir.RegisterList, +}; + +fn setFrameLoc( + func: *Func, + frame_index: FrameIndex, + base: Register, + offset: *i32, + comptime aligned: bool, +) void { + const frame_i = @intFromEnum(frame_index); + if (aligned) { + const alignment: InternPool.Alignment = func.frame_allocs.items(.abi_align)[frame_i]; + offset.* = math.sign(offset.*) * @as(i32, @intCast(alignment.backward(@intCast(@abs(offset.*))))); + } + func.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* }); + offset.* += func.frame_allocs.items(.abi_size)[frame_i]; +} + +fn computeFrameLayout(func: *Func) !FrameLayout { + const frame_allocs_len = func.frame_allocs.len; + try func.frame_locs.resize(func.gpa, frame_allocs_len); + const stack_frame_order = try func.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count); + defer func.gpa.free(stack_frame_order); + + const frame_size = func.frame_allocs.items(.abi_size); + const frame_align = func.frame_allocs.items(.abi_align); + + for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index| + frame_order.* = @enumFromInt(frame_index); + + { + const SortContext = struct { + frame_align: @TypeOf(frame_align), + pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool { + return context.frame_align[@intFromEnum(lhs)].compare(.gt, context.frame_align[@intFromEnum(rhs)]); + } + }; + const sort_context = SortContext{ .frame_align = frame_align }; + mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan); + } + + var save_reg_list = Mir.RegisterList{}; + for (abi.Registers.all_preserved) |reg| { + if (func.register_manager.isRegAllocated(reg)) { + save_reg_list.push(&abi.Registers.all_preserved, reg); + } + } + + const total_alloc_size: i32 = blk: { + var i: i32 = 0; + for (stack_frame_order) |frame_index| { + i += frame_size[@intFromEnum(frame_index)]; + } + break :blk i; + }; + + const saved_reg_size = save_reg_list.size(); + frame_size[@intFromEnum(FrameIndex.spill_frame)] = @intCast(saved_reg_size); + + // The total frame size is calculated by the amount of s registers you need to save * 8, as each + // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0 + // register. Finally we align the frame size to the alignment of the base pointer. + const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)]; + const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)]; + const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)]; + + // TODO: this 64 should be a 16, but we were clobbering the top and bottom of the frame. + // maybe everything can go from the bottom? + const acc_frame_size: i32 = std.mem.alignForward( + i32, + total_alloc_size + 64 + args_frame_size + spill_frame_size + call_frame_size, + @intCast(frame_align[@intFromEnum(FrameIndex.base_ptr)].toByteUnits().?), + ); + log.debug("frame size: {d}", .{acc_frame_size}); + + // store the ra at total_size - 8, so it's the very first thing in the stack + // relative to the fp + func.frame_locs.set( + @intFromEnum(FrameIndex.ret_addr), + .{ .base = .sp, .disp = acc_frame_size - 8 }, + ); + func.frame_locs.set( + @intFromEnum(FrameIndex.base_ptr), + .{ .base = .sp, .disp = acc_frame_size - 16 }, + ); + + // now we grow the stack frame from the bottom of total frame in order to + // not need to know the size of the first allocation. Stack offsets point at the "bottom" + // of variables. + var s0_offset: i32 = -acc_frame_size; + func.setFrameLoc(.stack_frame, .s0, &s0_offset, true); + for (stack_frame_order) |frame_index| func.setFrameLoc(frame_index, .s0, &s0_offset, true); + func.setFrameLoc(.args_frame, .s0, &s0_offset, true); + func.setFrameLoc(.call_frame, .s0, &s0_offset, true); + func.setFrameLoc(.spill_frame, .s0, &s0_offset, true); + + return .{ + .stack_adjust = @intCast(acc_frame_size), + .save_reg_list = save_reg_list, + }; +} + +fn ensureProcessDeathCapacity(func: *Func, additional_count: usize) !void { + const table = &func.branch_stack.items[func.branch_stack.items.len - 1].inst_table; + try table.ensureUnusedCapacity(func.gpa, additional_count); +} + +fn memSize(func: *Func, ty: Type) Memory.Size { + const pt = func.pt; + const zcu = pt.zcu; + return switch (ty.zigTypeTag(zcu)) { + .float => Memory.Size.fromBitSize(ty.floatBits(func.target)), + else => Memory.Size.fromByteSize(ty.abiSize(zcu)), + }; +} + +fn splitType(func: *Func, ty: Type) ![2]Type { + const zcu = func.pt.zcu; + const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none); + var parts: [2]Type = undefined; + if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| { + part.* = switch (class) { + .integer => switch (part_i) { + 0 => Type.u64, + 1 => part: { + const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; + const elem_ty = try func.pt.intType(.unsigned, @intCast(elem_size * 8)); + break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) { + 1 => elem_ty, + else => |len| try func.pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }), + }; + }, + else => unreachable, + }, + else => return func.fail("TODO: splitType class {}", .{class}), + }; + } else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts; + return func.fail("TODO implement splitType for {f}", .{ty.fmt(func.pt)}); +} + +/// Truncates the value in the register in place. +/// Clobbers any remaining bits. +fn truncateRegister(func: *Func, ty: Type, reg: Register) !void { + const pt = func.pt; + const zcu = pt.zcu; + const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{ + .signedness = .unsigned, + .bits = @intCast(ty.bitSize(zcu)), + }; + assert(reg.class() == .int); + + const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return; + switch (int_info.signedness) { + .signed => { + _ = try func.addInst(.{ + .tag = .slli, + + .data = .{ + .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(shift), + }, + }, + }); + _ = try func.addInst(.{ + .tag = .srai, + + .data = .{ + .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(shift), + }, + }, + }); + }, + .unsigned => { + const mask = ~@as(u64, 0) >> shift; + if (mask < 256) { + _ = try func.addInst(.{ + .tag = .andi, + + .data = .{ + .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(@intCast(mask)), + }, + }, + }); + } else { + _ = try func.addInst(.{ + .tag = .slli, + + .data = .{ + .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(shift), + }, + }, + }); + _ = try func.addInst(.{ + .tag = .srli, + + .data = .{ + .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(shift), + }, + }, + }); + } + }, + } +} + +fn allocFrameIndex(func: *Func, alloc: FrameAlloc) !FrameIndex { + const frame_allocs_slice = func.frame_allocs.slice(); + const frame_size = frame_allocs_slice.items(.abi_size); + const frame_align = frame_allocs_slice.items(.abi_align); + + const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)]; + stack_frame_align.* = stack_frame_align.max(alloc.abi_align); + + for (func.free_frame_indices.keys(), 0..) |frame_index, free_i| { + const abi_size = frame_size[@intFromEnum(frame_index)]; + if (abi_size != alloc.abi_size) continue; + const abi_align = &frame_align[@intFromEnum(frame_index)]; + abi_align.* = abi_align.max(alloc.abi_align); + + _ = func.free_frame_indices.swapRemoveAt(free_i); + return frame_index; + } + const frame_index: FrameIndex = @enumFromInt(func.frame_allocs.len); + try func.frame_allocs.append(func.gpa, alloc); + log.debug("allocated frame {}", .{frame_index}); + return frame_index; +} + +/// Use a pointer instruction as the basis for allocating stack memory. +fn allocMemPtr(func: *Func, inst: Air.Inst.Index) !FrameIndex { + const pt = func.pt; + const zcu = pt.zcu; + const ptr_ty = func.typeOfIndex(inst); + const val_ty = ptr_ty.childType(zcu); + return func.allocFrameIndex(FrameAlloc.init(.{ + .size = math.cast(u32, val_ty.abiSize(zcu)) orelse { + return func.fail("type '{f}' too big to fit into stack frame", .{val_ty.fmt(pt)}); + }, + .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"), + })); +} + +fn typeRegClass(func: *Func, ty: Type) abi.RegisterClass { + const pt = func.pt; + const zcu = pt.zcu; + return switch (ty.zigTypeTag(zcu)) { + .float => .float, + .vector => .vector, + else => .int, + }; +} + +fn regGeneralClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet { + return switch (ty.zigTypeTag(func.pt.zcu)) { + .float => abi.Registers.Float.general_purpose, + .vector => abi.Registers.Vector.general_purpose, + else => abi.Registers.Integer.general_purpose, + }; +} + +fn regTempClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet { + return switch (ty.zigTypeTag(func.pt.zcu)) { + .float => abi.Registers.Float.temporary, + .vector => abi.Registers.Vector.general_purpose, // there are no temporary vector registers + else => abi.Registers.Integer.temporary, + }; +} + +fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue { + const pt = func.pt; + const zcu = pt.zcu; + + const bit_size = elem_ty.bitSize(zcu); + const min_size: u64 = switch (elem_ty.zigTypeTag(zcu)) { + .float => if (func.hasFeature(.d)) 64 else 32, + .vector => 256, // TODO: calculate it from avl * vsew + else => 64, + }; + + if (reg_ok and bit_size <= min_size) { + if (func.register_manager.tryAllocReg(inst, func.regGeneralClassForType(elem_ty))) |reg| { + return .{ .register = reg }; + } + } else if (reg_ok and elem_ty.zigTypeTag(zcu) == .vector) { + return func.fail("did you forget to extend vector registers before allocating", .{}); + } + + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(elem_ty, zcu)); + return .{ .load_frame = .{ .index = frame_index } }; +} + +/// Allocates a register from the general purpose set and returns the Register and the Lock. +/// +/// Up to the caller to unlock the register later. +fn allocReg(func: *Func, reg_class: abi.RegisterClass) !struct { Register, RegisterLock } { + if (reg_class == .float and !func.hasFeature(.f)) + std.debug.panic("allocReg class == float where F isn't enabled", .{}); + if (reg_class == .vector and !func.hasFeature(.v)) + std.debug.panic("allocReg class == vector where V isn't enabled", .{}); + + const class = switch (reg_class) { + .int => abi.Registers.Integer.general_purpose, + .float => abi.Registers.Float.general_purpose, + .vector => abi.Registers.Vector.general_purpose, + }; + + const reg = try func.register_manager.allocReg(null, class); + const lock = func.register_manager.lockRegAssumeUnused(reg); + return .{ reg, lock }; +} + +/// Similar to `allocReg` but will copy the MCValue into the Register unless `operand` is already +/// a register, in which case it will return a possible lock to that register. +fn promoteReg(func: *Func, ty: Type, operand: MCValue) !struct { Register, ?RegisterLock } { + if (operand == .register) { + const op_reg = operand.register; + return .{ op_reg, func.register_manager.lockReg(operand.register) }; + } + + const class = func.typeRegClass(ty); + const reg, const lock = try func.allocReg(class); + try func.genSetReg(ty, reg, operand); + return .{ reg, lock }; +} + +fn elemOffset(func: *Func, index_ty: Type, index: MCValue, elem_size: u64) !Register { + const reg: Register = blk: { + switch (index) { + .immediate => |imm| { + // Optimisation: if index MCValue is an immediate, we can multiply in `comptime` + // and set the register directly to the scaled offset as an immediate. + const reg = try func.register_manager.allocReg(null, func.regGeneralClassForType(index_ty)); + try func.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size }); + break :blk reg; + }, + else => { + const reg = try func.copyToTmpRegister(index_ty, index); + const lock = func.register_manager.lockRegAssumeUnused(reg); + defer func.register_manager.unlockReg(lock); + + const result_reg, const result_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(result_lock); + + try func.genBinOp( + .mul, + .{ .register = reg }, + index_ty, + .{ .immediate = elem_size }, + index_ty, + result_reg, + ); + + break :blk result_reg; + }, + } + }; + return reg; +} + +pub fn spillInstruction(func: *Func, reg: Register, inst: Air.Inst.Index) !void { + const tracking = func.inst_tracking.getPtr(inst) orelse return; + for (tracking.getRegs()) |tracked_reg| { + if (tracked_reg.id() == reg.id()) break; + } else unreachable; // spilled reg not tracked with spilled instruciton + try tracking.spill(func, inst); + try tracking.trackSpill(func, inst); +} + +pub fn spillRegisters(func: *Func, comptime registers: []const Register) !void { + inline for (registers) |reg| try func.register_manager.getKnownReg(reg, null); +} + +/// Copies a value to a register without tracking the register. The register is not considered +/// allocated. A second call to `copyToTmpRegister` may return the same register. +/// This can have a side effect of spilling instructions to the stack to free up a register. +fn copyToTmpRegister(func: *Func, ty: Type, mcv: MCValue) !Register { + log.debug("copyToTmpRegister ty: {f}", .{ty.fmt(func.pt)}); + const reg = try func.register_manager.allocReg(null, func.regTempClassForType(ty)); + try func.genSetReg(ty, reg, mcv); + return reg; +} + +/// Allocates a new register and copies `mcv` into it. +/// `reg_owner` is the instruction that gets associated with the register in the register table. +/// This can have a side effect of spilling instructions to the stack to free up a register. +fn copyToNewRegister(func: *Func, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { + const ty = func.typeOfIndex(reg_owner); + const reg = try func.register_manager.allocReg(reg_owner, func.regGeneralClassForType(ty)); + try func.genSetReg(func.typeOfIndex(reg_owner), reg, mcv); + return MCValue{ .register = reg }; +} + +fn airAlloc(func: *Func, inst: Air.Inst.Index) !void { + const result = MCValue{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } }; + return func.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airRetPtr(func: *Func, inst: Air.Inst.Index) !void { + const result: MCValue = switch (func.ret_mcv.long) { + .none => .{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } }, + .load_frame => .{ .register_offset = .{ + .reg = (try func.copyToNewRegister( + inst, + func.ret_mcv.long, + )).register, + .off = func.ret_mcv.short.indirect.off, + } }, + else => |t| return func.fail("TODO: airRetPtr {s}", .{@tagName(t)}), + }; + return func.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airFptrunc(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFptrunc for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airFpext(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFpext for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIntCast(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const src_ty = func.typeOf(ty_op.operand); + const dst_ty = func.typeOfIndex(inst); + + const result: MCValue = result: { + const src_int_info = src_ty.intInfo(zcu); + const dst_int_info = dst_ty.intInfo(zcu); + + const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; + + const src_mcv = try func.resolveInst(ty_op.operand); + + const src_storage_bits: u16 = switch (src_mcv) { + .register => 64, + .load_frame => src_int_info.bits, + else => return func.fail("airIntCast from {s}", .{@tagName(src_mcv)}), + }; + + const dst_mcv = if (dst_int_info.bits <= src_storage_bits and + math.divCeil(u16, dst_int_info.bits, 64) catch unreachable == + math.divCeil(u32, src_storage_bits, 64) catch unreachable and + func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { + const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true); + try func.genCopy(min_ty, dst_mcv, src_mcv); + break :dst dst_mcv; + }; + + if (dst_int_info.bits <= src_int_info.bits) + break :result dst_mcv; + + if (dst_int_info.bits > 64 or src_int_info.bits > 64) + break :result null; // TODO + + break :result dst_mcv; + } orelse return func.fail("TODO: implement airIntCast from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airTrunc(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + if (func.liveness.isUnused(inst)) + return func.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none }); + // we assume no zeroext in the "Zig ABI", so it's fine to just not truncate it. + const operand = try func.resolveInst(ty_op.operand); + + // we can do it just to be safe, but this shouldn't be needed for no-runtime safety modes + switch (operand) { + .register => |reg| try func.truncateRegister(func.typeOf(ty_op.operand), reg), + else => {}, + } + + return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none }); +} + +fn airNot(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const pt = func.pt; + const zcu = pt.zcu; + + const operand = try func.resolveInst(ty_op.operand); + const ty = func.typeOf(ty_op.operand); + + const operand_reg, const operand_lock = try func.promoteReg(ty, operand); + defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); + + const dst_reg: Register = + if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register) + operand.register + else + (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register; + + switch (ty.zigTypeTag(zcu)) { + .bool => { + _ = try func.addInst(.{ + .tag = .pseudo_not, + .data = .{ + .rr = .{ + .rs = operand_reg, + .rd = dst_reg, + }, + }, + }); + }, + .int => { + const size = ty.bitSize(zcu); + if (!math.isPowerOfTwo(size)) + return func.fail("TODO: airNot non-pow 2 int size", .{}); + + switch (size) { + 32, 64 => { + _ = try func.addInst(.{ + .tag = .xori, + .data = .{ + .i_type = .{ + .rd = dst_reg, + .rs1 = operand_reg, + .imm12 = Immediate.s(-1), + }, + }, + }); + }, + 8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}), + else => unreachable, + } + }, + else => unreachable, + } + + break :result .{ .register = dst_reg }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSlice(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data; + + const slice_ty = func.typeOfIndex(inst); + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu)); + + const ptr_ty = func.typeOf(bin_op.lhs); + try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }); + + const len_ty = func.typeOf(bin_op.rhs); + try func.genSetMem( + .{ .frame = frame_index }, + @intCast(ptr_ty.abiSize(zcu)), + len_ty, + .{ .air_ref = bin_op.rhs }, + ); + + const result = MCValue{ .load_frame = .{ .index = frame_index } }; + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airBinOp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const pt = func.pt; + const zcu = pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); + + const dst_ty = func.typeOfIndex(inst); + if (dst_ty.isAbiInt(zcu)) { + const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + const bit_size: u32 = @intCast(dst_ty.bitSize(zcu)); + if (abi_size * 8 > bit_size) { + const dst_lock = switch (dst_mcv) { + .register => |dst_reg| func.register_manager.lockRegAssumeUnused(dst_reg), + else => null, + }; + defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); + + if (dst_mcv.isRegister()) { + try func.truncateRegister(dst_ty, dst_mcv.getReg().?); + } else { + const tmp_reg, const tmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(tmp_lock); + + const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1)); + const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref(); + try func.genSetReg(hi_ty, tmp_reg, hi_mcv); + try func.truncateRegister(dst_ty, tmp_reg); + try func.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg }); + } + } + } + + return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn binOp( + func: *Func, + maybe_inst: ?Air.Inst.Index, + air_tag: Air.Inst.Tag, + lhs_air: Air.Inst.Ref, + rhs_air: Air.Inst.Ref, +) !MCValue { + _ = maybe_inst; + const pt = func.pt; + const zcu = pt.zcu; + const lhs_ty = func.typeOf(lhs_air); + const rhs_ty = func.typeOf(rhs_air); + + if (lhs_ty.isRuntimeFloat()) libcall: { + const float_bits = lhs_ty.floatBits(func.target); + const type_needs_libcall = switch (float_bits) { + 16 => true, + 32, 64 => false, + 80, 128 => true, + else => unreachable, + }; + if (!type_needs_libcall) break :libcall; + return func.fail("binOp libcall runtime-float ops", .{}); + } + + // don't have support for certain sizes of addition + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => {}, // works differently and fails in a different place + else => if (lhs_ty.bitSize(zcu) > 64) return func.fail("TODO: binOp >= 64 bits", .{}), + } + + const lhs_mcv = try func.resolveInst(lhs_air); + const rhs_mcv = try func.resolveInst(rhs_air); + + const class_for_dst_ty: abi.RegisterClass = switch (air_tag) { + // will always return int register no matter the input + .cmp_eq, + .cmp_neq, + .cmp_lt, + .cmp_lte, + .cmp_gt, + .cmp_gte, + => .int, + + else => func.typeRegClass(lhs_ty), + }; + + const dst_reg, const dst_lock = try func.allocReg(class_for_dst_ty); + defer func.register_manager.unlockReg(dst_lock); + + try func.genBinOp( + air_tag, + lhs_mcv, + lhs_ty, + rhs_mcv, + rhs_ty, + dst_reg, + ); + + return .{ .register = dst_reg }; +} + +/// Does the same thing as binOp however is meant to be used internally to the backend. +/// +/// The `dst_reg` argument is meant to be caller-locked. Asserts that the binOp result can be +/// fit into the register. +/// +/// Assumes that the `dst_reg` class is correct. +fn genBinOp( + func: *Func, + tag: Air.Inst.Tag, + lhs_mcv: MCValue, + lhs_ty: Type, + rhs_mcv: MCValue, + rhs_ty: Type, + dst_reg: Register, +) !void { + const pt = func.pt; + const zcu = pt.zcu; + const bit_size = lhs_ty.bitSize(zcu); + + const is_unsigned = lhs_ty.isUnsignedInt(zcu); + + const lhs_reg, const maybe_lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv); + const rhs_reg, const maybe_rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv); + + defer if (maybe_lhs_lock) |lock| func.register_manager.unlockReg(lock); + defer if (maybe_rhs_lock) |lock| func.register_manager.unlockReg(lock); + + switch (tag) { + .add, + .add_wrap, + .sub, + .sub_wrap, + .mul, + .mul_wrap, + .rem, + .div_trunc, + .div_exact, + => { + switch (tag) { + .rem, + .div_trunc, + .div_exact, + => { + if (!math.isPowerOfTwo(bit_size)) { + try func.truncateRegister(lhs_ty, lhs_reg); + try func.truncateRegister(rhs_ty, rhs_reg); + } + }, + else => { + if (!math.isPowerOfTwo(bit_size)) + return func.fail( + "TODO: genBinOp verify if needs to truncate {s} non-pow 2, found {}", + .{ @tagName(tag), bit_size }, + ); + }, + } + + switch (lhs_ty.zigTypeTag(zcu)) { + .int => { + const mnem: Mnemonic = switch (tag) { + .add, .add_wrap => switch (bit_size) { + 8, 16, 64 => .add, + 32 => .addw, + else => unreachable, + }, + .sub, .sub_wrap => switch (bit_size) { + 8, 16, 32 => .subw, + 64 => .sub, + else => unreachable, + }, + .mul, .mul_wrap => switch (bit_size) { + 8, 16, 64 => .mul, + 32 => .mulw, + else => unreachable, + }, + .rem => switch (bit_size) { + 8, 16, 32 => if (is_unsigned) .remuw else .remw, + else => if (is_unsigned) .remu else .rem, + }, + .div_trunc, .div_exact => switch (bit_size) { + 8, 16, 32 => if (is_unsigned) .divuw else .divw, + else => if (is_unsigned) .divu else .div, + }, + else => unreachable, + }; + + _ = try func.addInst(.{ + .tag = mnem, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + }, + }, + }); + }, + .float => { + const mir_tag: Mnemonic = switch (tag) { + .add => switch (bit_size) { + 32 => .fadds, + 64 => .faddd, + else => unreachable, + }, + .sub => switch (bit_size) { + 32 => .fsubs, + 64 => .fsubd, + else => unreachable, + }, + .mul => switch (bit_size) { + 32 => .fmuls, + 64 => .fmuld, + else => unreachable, + }, + else => return func.fail("TODO: genBinOp {s} Float", .{@tagName(tag)}), + }; + + _ = try func.addInst(.{ + .tag = mir_tag, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + }, + }, + }); + }, + .vector => { + const num_elem = lhs_ty.vectorLen(zcu); + const elem_size = lhs_ty.childType(zcu).bitSize(zcu); + + const child_ty = lhs_ty.childType(zcu); + + const mir_tag: Mnemonic = switch (tag) { + .add => switch (child_ty.zigTypeTag(zcu)) { + .int => .vaddvv, + .float => .vfaddvv, + else => unreachable, + }, + .sub => switch (child_ty.zigTypeTag(zcu)) { + .int => .vsubvv, + .float => .vfsubvv, + else => unreachable, + }, + .mul => switch (child_ty.zigTypeTag(zcu)) { + .int => .vmulvv, + .float => .vfmulvv, + else => unreachable, + }, + else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}), + }; + + try func.setVl(.zero, num_elem, .{ + .vsew = switch (elem_size) { + 8 => .@"8", + 16 => .@"16", + 32 => .@"32", + 64 => .@"64", + else => return func.fail("TODO: genBinOp > 64 bit elements, found {d}", .{elem_size}), + }, + .vlmul = .m1, + .vma = true, + .vta = true, + }); + + _ = try func.addInst(.{ + .tag = mir_tag, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = rhs_reg, + .rs2 = lhs_reg, + }, + }, + }); + }, + else => unreachable, + } + }, + + .add_sat, + => { + if (bit_size != 64 or !is_unsigned) + return func.fail("TODO: genBinOp ty: {f}", .{lhs_ty.fmt(pt)}); + + const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg }); + const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg); + defer func.register_manager.unlockReg(tmp_lock); + + _ = try func.addInst(.{ + .tag = .add, + .data = .{ .r_type = .{ + .rd = tmp_reg, + .rs1 = rhs_reg, + .rs2 = lhs_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .sltu, + .data = .{ .r_type = .{ + .rd = dst_reg, + .rs1 = tmp_reg, + .rs2 = lhs_reg, + } }, + }); + + // neg dst_reg, dst_reg + _ = try func.addInst(.{ + .tag = .sub, + .data = .{ .r_type = .{ + .rd = dst_reg, + .rs1 = .zero, + .rs2 = dst_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .@"or", + .data = .{ .r_type = .{ + .rd = dst_reg, + .rs1 = dst_reg, + .rs2 = tmp_reg, + } }, + }); + }, + + .ptr_add, + .ptr_sub, + => { + const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg }); + const tmp_mcv = MCValue{ .register = tmp_reg }; + const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg); + defer func.register_manager.unlockReg(tmp_lock); + + // RISC-V has no immediate mul, so we copy the size to a temporary register + const elem_size = lhs_ty.elemType2(zcu).abiSize(zcu); + const elem_size_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = elem_size }); + + try func.genBinOp( + .mul, + tmp_mcv, + rhs_ty, + .{ .register = elem_size_reg }, + Type.u64, + tmp_reg, + ); + + try func.genBinOp( + switch (tag) { + .ptr_add => .add, + .ptr_sub => .sub, + else => unreachable, + }, + lhs_mcv, + Type.u64, // we know it's a pointer, so it'll be usize. + tmp_mcv, + Type.u64, + dst_reg, + ); + }, + + .bit_and, + .bit_or, + .bool_and, + .bool_or, + => { + _ = try func.addInst(.{ + .tag = switch (tag) { + .bit_and, .bool_and => .@"and", + .bit_or, .bool_or => .@"or", + else => unreachable, + }, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + }, + }, + }); + + switch (tag) { + .bool_and, + .bool_or, + => try func.truncateRegister(Type.bool, dst_reg), + else => {}, + } + }, + + .shr, + .shr_exact, + .shl, + .shl_exact, + => { + if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) return func.fail("TODO: vector shift with scalar rhs", .{}); + if (bit_size > 64) return func.fail("TODO: genBinOp shift > 64 bits, {}", .{bit_size}); + try func.truncateRegister(rhs_ty, rhs_reg); + + const mir_tag: Mnemonic = switch (tag) { + .shl, .shl_exact => switch (bit_size) { + 1...31, 33...64 => .sll, + 32 => .sllw, + else => unreachable, + }, + .shr, .shr_exact => switch (bit_size) { + 1...31, 33...64 => .srl, + 32 => .srlw, + else => unreachable, + }, + else => unreachable, + }; + + _ = try func.addInst(.{ + .tag = mir_tag, + .data = .{ .r_type = .{ + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + } }, + }); + }, + + // TODO: move the isel logic out of lower and into here. + .cmp_eq, + .cmp_neq, + .cmp_lt, + .cmp_lte, + .cmp_gt, + .cmp_gte, + => { + assert(lhs_reg.class() == rhs_reg.class()); + if (lhs_reg.class() == .int) { + try func.truncateRegister(lhs_ty, lhs_reg); + try func.truncateRegister(rhs_ty, rhs_reg); + } + + _ = try func.addInst(.{ + .tag = .pseudo_compare, + .data = .{ + .compare = .{ + .op = switch (tag) { + .cmp_eq => .eq, + .cmp_neq => .neq, + .cmp_lt => .lt, + .cmp_lte => .lte, + .cmp_gt => .gt, + .cmp_gte => .gte, + else => unreachable, + }, + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + .ty = lhs_ty, + }, + }, + }); + }, + + // A branchless @min/@max sequence. + // + // Assume that a0 and a1 are the lhs and rhs respectively. + // Also assume that a2 is the destination register. + // + // Algorithm: + // slt s0, a0, a1 + // sub s0, zero, s0 + // xor a2, a0, a1 + // and s0, a2, s0 + // xor a2, a0, s0 # a0 is @min, a1 is @max + // + // "slt s0, a0, a1" will set s0 to 1 if a0 is less than a1, and 1 otherwise. + // + // "sub s0, zero, s0" will set all the bits of s0 to 1 if it was 1, otherwise it'll remain at 0. + // + // "xor a2, a0, a1" stores the bitwise XOR of a0 and a1 in a2. Effectively getting the difference between them. + // + // "and a0, a2, s0" here we mask the result of the XOR with the negated s0. If a0 < a1, s0 is -1, which + // doesn't change the bits of a2. If a0 >= a1, s0 is 0, nullifying a2. + // + // "xor a2, a0, s0" the final XOR operation adjusts a2 to be the minimum value of a0 and a1. If a0 was less than + // a1, s0 was -1, flipping all the bits in a2 and effectively restoring a0. If a0 was greater than or equal to a1, + // s0 was 0, leaving a2 unchanged as a0. + .min, .max => { + switch (lhs_ty.zigTypeTag(zcu)) { + .int => { + const int_info = lhs_ty.intInfo(zcu); + + const mask_reg, const mask_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(mask_lock); + + _ = try func.addInst(.{ + .tag = if (int_info.signedness == .unsigned) .sltu else .slt, + .data = .{ .r_type = .{ + .rd = mask_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .sub, + .data = .{ .r_type = .{ + .rd = mask_reg, + .rs1 = .zero, + .rs2 = mask_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .xor, + .data = .{ .r_type = .{ + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .@"and", + .data = .{ .r_type = .{ + .rd = mask_reg, + .rs1 = dst_reg, + .rs2 = mask_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .xor, + .data = .{ .r_type = .{ + .rd = dst_reg, + .rs1 = if (tag == .min) rhs_reg else lhs_reg, + .rs2 = mask_reg, + } }, + }); + }, + else => |t| return func.fail("TODO: genBinOp min/max for {s}", .{@tagName(t)}), + } + }, + else => return func.fail("TODO: genBinOp {}", .{tag}), + } +} + +fn airPtrArithmetic(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data; + const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); + return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airAddWithOverflow(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; + + const rhs_ty = func.typeOf(extra.rhs); + const lhs_ty = func.typeOf(extra.lhs); + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => return func.fail("TODO implement add with overflow for Vector type", .{}), + .int => { + const int_info = lhs_ty.intInfo(zcu); + + const tuple_ty = func.typeOfIndex(inst); + const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); + const offset = result_mcv.load_frame; + + if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { + const add_result = try func.binOp(null, .add, extra.lhs, extra.rhs); + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), + lhs_ty, + add_result, + ); + + const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result); + const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg); + defer func.register_manager.unlockReg(trunc_reg_lock); + + const overflow_reg, const overflow_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(overflow_lock); + + // if the result isn't equal after truncating it to the given type, + // an overflow must have happened. + try func.truncateRegister(lhs_ty, trunc_reg); + try func.genBinOp( + .cmp_neq, + add_result, + lhs_ty, + .{ .register = trunc_reg }, + rhs_ty, + overflow_reg, + ); + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), + Type.u1, + .{ .register = overflow_reg }, + ); + + break :result result_mcv; + } else { + const rhs_mcv = try func.resolveInst(extra.rhs); + const lhs_mcv = try func.resolveInst(extra.lhs); + + const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv); + const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv); + defer { + if (rhs_lock) |lock| func.register_manager.unlockReg(lock); + if (lhs_lock) |lock| func.register_manager.unlockReg(lock); + } + + try func.truncateRegister(rhs_ty, rhs_reg); + try func.truncateRegister(lhs_ty, lhs_reg); + + const dest_reg, const dest_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(dest_lock); + + _ = try func.addInst(.{ + .tag = .add, + .data = .{ .r_type = .{ + .rs1 = rhs_reg, + .rs2 = lhs_reg, + .rd = dest_reg, + } }, + }); + + try func.truncateRegister(func.typeOfIndex(inst), dest_reg); + const add_result: MCValue = .{ .register = dest_reg }; + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), + lhs_ty, + add_result, + ); + + const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result); + const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg); + defer func.register_manager.unlockReg(trunc_reg_lock); + + const overflow_reg, const overflow_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(overflow_lock); + + // if the result isn't equal after truncating it to the given type, + // an overflow must have happened. + try func.truncateRegister(lhs_ty, trunc_reg); + try func.genBinOp( + .cmp_neq, + add_result, + lhs_ty, + .{ .register = trunc_reg }, + rhs_ty, + overflow_reg, + ); + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), + Type.u1, + .{ .register = overflow_reg }, + ); + + break :result result_mcv; + } + }, + else => unreachable, + } + }; + + return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airSubWithOverflow(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const lhs = try func.resolveInst(extra.lhs); + const rhs = try func.resolveInst(extra.rhs); + const lhs_ty = func.typeOf(extra.lhs); + const rhs_ty = func.typeOf(extra.rhs); + + const int_info = lhs_ty.intInfo(zcu); + + if (!math.isPowerOfTwo(int_info.bits) or int_info.bits < 8) { + return func.fail("TODO: airSubWithOverflow non-power of 2 and less than 8 bits", .{}); + } + + if (int_info.bits > 64) { + return func.fail("TODO: airSubWithOverflow > 64 bits", .{}); + } + + const tuple_ty = func.typeOfIndex(inst); + const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); + const offset = result_mcv.load_frame; + + const dest_mcv = try func.binOp(null, .sub, extra.lhs, extra.rhs); + assert(dest_mcv == .register); + const dest_reg = dest_mcv.register; + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), + lhs_ty, + .{ .register = dest_reg }, + ); + + const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs); + defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock); + + const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs); + defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock); + + const overflow_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = 0 }); + + const overflow_lock = func.register_manager.lockRegAssumeUnused(overflow_reg); + defer func.register_manager.unlockReg(overflow_lock); + + switch (int_info.signedness) { + .unsigned => { + _ = try func.addInst(.{ + .tag = .sltu, + .data = .{ .r_type = .{ + .rd = overflow_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + } }, + }); + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), + Type.u1, + .{ .register = overflow_reg }, + ); + + break :result result_mcv; + }, + .signed => { + switch (int_info.bits) { + 64 => { + _ = try func.addInst(.{ + .tag = .slt, + .data = .{ .r_type = .{ + .rd = overflow_reg, + .rs1 = overflow_reg, + .rs2 = rhs_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .slt, + .data = .{ .r_type = .{ + .rd = rhs_reg, + .rs1 = rhs_reg, + .rs2 = lhs_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .xor, + .data = .{ .r_type = .{ + .rd = lhs_reg, + .rs1 = overflow_reg, + .rs2 = rhs_reg, + } }, + }); + + try func.genBinOp( + .cmp_neq, + .{ .register = overflow_reg }, + Type.u64, + .{ .register = rhs_reg }, + Type.u64, + overflow_reg, + ); + + try func.genSetMem( + .{ .frame = offset.index }, + offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), + Type.u1, + .{ .register = overflow_reg }, + ); + + break :result result_mcv; + }, + else => |int_bits| return func.fail("TODO: airSubWithOverflow signed {}", .{int_bits}), + } + }, + } + }; + + return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airMulWithOverflow(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const lhs = try func.resolveInst(extra.lhs); + const rhs = try func.resolveInst(extra.rhs); + const lhs_ty = func.typeOf(extra.lhs); + const rhs_ty = func.typeOf(extra.rhs); + + const tuple_ty = func.typeOfIndex(inst); + + // genSetReg needs to support register_offset src_mcv for this to be true. + const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); + + const result_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu)); + const overflow_off: i32 = @intCast(tuple_ty.structFieldOffset(1, zcu)); + + const dest_reg, const dest_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(dest_lock); + + try func.genBinOp( + .mul, + lhs, + lhs_ty, + rhs, + rhs_ty, + dest_reg, + ); + + try func.genCopy( + lhs_ty, + result_mcv.offset(result_off), + .{ .register = dest_reg }, + ); + + switch (lhs_ty.zigTypeTag(zcu)) { + else => |x| return func.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}), + .int => { + if (std.debug.runtime_safety) assert(lhs_ty.eql(rhs_ty, zcu)); + + const trunc_reg = try func.copyToTmpRegister(lhs_ty, .{ .register = dest_reg }); + const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg); + defer func.register_manager.unlockReg(trunc_reg_lock); + + const overflow_reg, const overflow_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(overflow_lock); + + // if the result isn't equal after truncating it to the given type, + // an overflow must have happened. + try func.truncateRegister(func.typeOf(extra.lhs), trunc_reg); + try func.genBinOp( + .cmp_neq, + .{ .register = dest_reg }, + lhs_ty, + .{ .register = trunc_reg }, + rhs_ty, + overflow_reg, + ); + + try func.genCopy( + lhs_ty, + result_mcv.offset(overflow_off), + .{ .register = overflow_reg }, + ); + + break :result result_mcv; + }, + } + }; + + return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airShlWithOverflow(func: *Func, inst: Air.Inst.Index) !void { + const zcu = func.pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (func.liveness.isUnused(inst)) + .unreach + else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu)) + return func.fail("TODO implement vector airShlWithOverflow with scalar rhs", .{}) + else + return func.fail("TODO implement airShlWithOverflow", .{}); + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSubSat(func: *Func, inst: Air.Inst.Index) !void { + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSubSat", .{}); + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMulSat(func: *Func, inst: Air.Inst.Index) !void { + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airMulSat", .{}); + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airShlSat(func: *Func, inst: Air.Inst.Index) !void { + const zcu = func.pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (func.liveness.isUnused(inst)) + .unreach + else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu)) + return func.fail("TODO implement vector airShlSat with scalar rhs", .{}) + else + return func.fail("TODO implement airShlSat", .{}); + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airOptionalPayload(func: *Func, inst: Air.Inst.Index) !void { + const zcu = func.pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = result: { + const pl_ty = func.typeOfIndex(inst); + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const opt_mcv = try func.resolveInst(ty_op.operand); + if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) { + switch (opt_mcv) { + .register => |pl_reg| try func.truncateRegister(pl_ty, pl_reg), + else => {}, + } + break :result opt_mcv; + } + + const pl_mcv = try func.allocRegOrMem(pl_ty, inst, true); + try func.genCopy(pl_ty, pl_mcv, opt_mcv); + break :result pl_mcv; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayloadPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement .optional_payload_ptr for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void { + const zcu = func.pt.zcu; + + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const dst_ty = func.typeOfIndex(inst); + const src_ty = func.typeOf(ty_op.operand); + const opt_ty = src_ty.childType(zcu); + const src_mcv = try func.resolveInst(ty_op.operand); + + if (opt_ty.optionalReprIsPayload(zcu)) { + break :result if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try func.copyToNewRegister(inst, src_mcv); + } + + const dst_mcv: MCValue = if (src_mcv.isRegister() and + func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try func.copyToNewRegister(inst, src_mcv); + + const pl_ty = dst_ty.childType(zcu); + const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); + try func.genSetMem( + .{ .reg = dst_mcv.getReg().? }, + pl_abi_size, + Type.bool, + .{ .immediate = 1 }, + ); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airUnwrapErrErr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const pt = func.pt; + const zcu = pt.zcu; + const err_union_ty = func.typeOf(ty_op.operand); + const err_ty = err_union_ty.errorUnionSet(zcu); + const payload_ty = err_union_ty.errorUnionPayload(zcu); + const operand = try func.resolveInst(ty_op.operand); + + const result: MCValue = result: { + if (err_ty.errorSetIsEmpty(zcu)) { + break :result .{ .immediate = 0 }; + } + + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :result operand; + } + + const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu)); + + switch (operand) { + .register => |reg| { + const eu_lock = func.register_manager.lockReg(reg); + defer if (eu_lock) |lock| func.register_manager.unlockReg(lock); + + const result = try func.copyToNewRegister(inst, operand); + if (err_off > 0) { + try func.genBinOp( + .shr, + result, + err_union_ty, + .{ .immediate = @as(u6, @intCast(err_off * 8)) }, + Type.u8, + result.register, + ); + } + break :result result; + }, + .load_frame => |frame_addr| break :result .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + @as(i32, @intCast(err_off)), + } }, + else => return func.fail("TODO implement unwrap_err_err for {}", .{operand}), + } + }; + + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airUnwrapErrPayload(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand_ty = func.typeOf(ty_op.operand); + const operand = try func.resolveInst(ty_op.operand); + const result = try func.genUnwrapErrUnionPayloadMir(operand_ty, operand); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn genUnwrapErrUnionPayloadMir( + func: *Func, + err_union_ty: Type, + err_union: MCValue, +) !MCValue { + const pt = func.pt; + const zcu = pt.zcu; + const payload_ty = err_union_ty.errorUnionPayload(zcu); + + const result: MCValue = result: { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); + switch (err_union) { + .load_frame => |frame_addr| break :result .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + payload_off, + } }, + .register => |reg| { + const eu_lock = func.register_manager.lockReg(reg); + defer if (eu_lock) |lock| func.register_manager.unlockReg(lock); + + const result_reg = try func.copyToTmpRegister(err_union_ty, err_union); + if (payload_off > 0) { + try func.genBinOp( + .shr, + .{ .register = result_reg }, + err_union_ty, + .{ .immediate = @as(u6, @intCast(payload_off * 8)) }, + Type.u8, + result_reg, + ); + } + break :result .{ .register = result_reg }; + }, + else => return func.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}), + } + }; + + return result; +} + +// *(E!T) -> E +fn airUnwrapErrErrPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union error ptr for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +// *(E!T) -> *T +fn airUnwrapErrPayloadPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union payload ptr for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +// *(E!T) => *T +fn airErrUnionPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const zcu = func.pt.zcu; + const src_ty = func.typeOf(ty_op.operand); + const src_mcv = try func.resolveInst(ty_op.operand); + + // `src_reg` contains the pointer to the error union + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try func.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = func.register_manager.lockRegAssumeUnused(src_reg); + defer func.register_manager.unlockReg(src_lock); + + // we set the place of where the error would have been to 0 + const eu_ty = src_ty.childType(zcu); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + try func.genSetMem(.{ .reg = src_reg }, err_off, err_ty, .{ .immediate = 0 }); + + const dst_reg, const dst_lock = if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + .{ src_reg, null } + else + try func.allocReg(.int); + defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); + + // move the pointer to be at the payload + const pl_off = errUnionPayloadOffset(pl_ty, zcu); + try func.genBinOp( + .add, + .{ .register = src_reg }, + Type.u64, + .{ .immediate = pl_off }, + Type.u64, + dst_reg, + ); + + break :result .{ .register = dst_reg }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void { + const result: MCValue = if (func.liveness.isUnused(inst)) + .unreach + else + return func.fail("TODO implement airErrReturnTrace for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airSetErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement airSetErrReturnTrace for {}", .{func.target.cpu.arch}); +} + +fn airSaveErrReturnTraceIndex(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{func.target.cpu.arch}); +} + +fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = result: { + const pl_ty = func.typeOf(ty_op.operand); + if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 }; + + const opt_ty = func.typeOfIndex(inst); + const pl_mcv = try func.resolveInst(ty_op.operand); + const same_repr = opt_ty.optionalReprIsPayload(zcu); + if (same_repr and func.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv; + + const pl_lock: ?RegisterLock = switch (pl_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (pl_lock) |lock| func.register_manager.unlockReg(lock); + + const opt_mcv = try func.allocRegOrMem(opt_ty, inst, false); + try func.genCopy(pl_ty, opt_mcv, pl_mcv); + + if (!same_repr) { + const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); + switch (opt_mcv) { + .load_frame => |frame_addr| { + try func.genCopy(pl_ty, opt_mcv, pl_mcv); + try func.genSetMem( + .{ .frame = frame_addr.index }, + frame_addr.off + pl_abi_size, + Type.u8, + .{ .immediate = 1 }, + ); + }, + else => unreachable, + } + } + break :result opt_mcv; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +/// T to E!T +fn airWrapErrUnionPayload(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const eu_ty = ty_op.ty.toType(); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + const operand = try func.resolveInst(ty_op.operand); + + const result: MCValue = result: { + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 }; + + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu)); + const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand); + try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }); + break :result .{ .load_frame = .{ .index = frame_index } }; + }; + + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +/// E to E!T +fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const eu_ty = ty_op.ty.toType(); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + + const result: MCValue = result: { + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try func.resolveInst(ty_op.operand); + + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu)); + const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null }); + const operand = try func.resolveInst(ty_op.operand); + try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand); + break :result .{ .load_frame = .{ .index = frame_index } }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airRuntimeNavPtr(func: *Func, inst: Air.Inst.Index) !void { + const zcu = func.pt.zcu; + const ip = &zcu.intern_pool; + const ty_nav = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; + const ptr_ty: Type = .fromInterned(ty_nav.ty); + + const nav = ip.getNav(ty_nav.nav); + const tlv_sym_index = if (func.bin_file.cast(.elf)) |elf_file| sym: { + const zo = elf_file.zigObjectPtr().?; + if (nav.getExtern(ip)) |e| { + break :sym try elf_file.getGlobalSymbol(nav.name.toSlice(ip), e.lib_name.toSlice(ip)); + } + break :sym try zo.getOrCreateMetadataForNav(zcu, ty_nav.nav); + } else return func.fail("TODO runtime_nav_ptr on {}", .{func.bin_file.tag}); + + const dest_mcv = try func.allocRegOrMem(ptr_ty, inst, true); + if (dest_mcv.isRegister()) { + _ = try func.addInst(.{ + .tag = .pseudo_load_tlv, + .data = .{ .reloc = .{ + .register = dest_mcv.getReg().?, + .atom_index = try func.owner.getSymbolIndex(func), + .sym_index = tlv_sym_index, + } }, + }); + } else { + const tmp_reg, const tmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(tmp_lock); + _ = try func.addInst(.{ + .tag = .pseudo_load_tlv, + .data = .{ .reloc = .{ + .register = tmp_reg, + .atom_index = try func.owner.getSymbolIndex(func), + .sym_index = tlv_sym_index, + } }, + }); + try func.genCopy(ptr_ty, dest_mcv, .{ .register = tmp_reg }); + } + + return func.finishAir(inst, dest_mcv, .{ .none, .none, .none }); +} + +fn airTry(func: *Func, inst: Air.Inst.Index) !void { + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = func.air.extraData(Air.Try, pl_op.payload); + const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]); + const operand_ty = func.typeOf(pl_op.operand); + const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false); + return func.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn genTry( + func: *Func, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + body: []const Air.Inst.Index, + operand_ty: Type, + operand_is_ptr: bool, +) !MCValue { + _ = operand_is_ptr; + + const liveness_cond_br = func.liveness.getCondBr(inst); + + const operand_mcv = try func.resolveInst(operand); + const is_err_mcv = try func.isErr(null, operand_ty, operand_mcv); + + // A branch to the false section. Uses beq. 1 is the default "true" state. + const reloc = try func.condBr(Type.anyerror, is_err_mcv); + + if (func.liveness.operandDies(inst, 0)) { + if (operand.toIndex()) |operand_inst| try func.processDeath(operand_inst); + } + + func.scope_generation += 1; + const state = try func.saveState(); + + for (liveness_cond_br.else_deaths) |death| try func.processDeath(death); + try func.genBody(body); + try func.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + func.performReloc(reloc); + + for (liveness_cond_br.then_deaths) |death| try func.processDeath(death); + + const result = if (func.liveness.isUnused(inst)) + .unreach + else + try func.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv); + + return result; +} + +fn airSlicePtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = result: { + const src_mcv = try func.resolveInst(ty_op.operand); + if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; + + const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + const dst_ty = func.typeOfIndex(inst); + try func.genCopy(dst_ty, dst_mcv, src_mcv); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSliceLen(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const src_mcv = try func.resolveInst(ty_op.operand); + const ty = func.typeOfIndex(inst); + + switch (src_mcv) { + .load_frame => |frame_addr| { + const len_mcv: MCValue = .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + 8, + } }; + if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv; + + const dst_mcv = try func.allocRegOrMem(ty, inst, true); + try func.genCopy(Type.u64, dst_mcv, len_mcv); + break :result dst_mcv; + }, + .register_pair => |pair| { + const len_mcv: MCValue = .{ .register = pair[1] }; + + if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv; + + const dst_mcv = try func.allocRegOrMem(ty, inst, true); + try func.genCopy(Type.u64, dst_mcv, len_mcv); + break :result dst_mcv; + }, + else => return func.fail("TODO airSliceLen for {}", .{src_mcv}), + } + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPtrSliceLenPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const src_mcv = try func.resolveInst(ty_op.operand); + + const dst_reg, const dst_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(dst_lock); + const dst_mcv: MCValue = .{ .register = dst_reg }; + + try func.genCopy(Type.u64, dst_mcv, src_mcv.offset(8)); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPtrSlicePtrPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const opt_mcv = try func.resolveInst(ty_op.operand); + const dst_mcv = if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) + opt_mcv + else + try func.copyToNewRegister(inst, opt_mcv); + return func.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airSliceElemVal(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const result: MCValue = result: { + const elem_ty = func.typeOfIndex(inst); + if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const slice_ty = func.typeOf(bin_op.lhs); + const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); + const elem_ptr = try func.genSliceElemPtr(bin_op.lhs, bin_op.rhs); + const dst_mcv = try func.allocRegOrMem(elem_ty, inst, false); + try func.load(dst_mcv, elem_ptr, slice_ptr_field_type); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSliceElemPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; + const dst_mcv = try func.genSliceElemPtr(extra.lhs, extra.rhs); + return func.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); +} + +fn genSliceElemPtr(func: *Func, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { + const pt = func.pt; + const zcu = pt.zcu; + const slice_ty = func.typeOf(lhs); + const slice_mcv = try func.resolveInst(lhs); + const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (slice_mcv_lock) |lock| func.register_manager.unlockReg(lock); + + const elem_ty = slice_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + const index_ty = func.typeOf(rhs); + const index_mcv = try func.resolveInst(rhs); + const index_mcv_lock: ?RegisterLock = switch (index_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_mcv_lock) |lock| func.register_manager.unlockReg(lock); + + const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_size); + const offset_reg_lock = func.register_manager.lockRegAssumeUnused(offset_reg); + defer func.register_manager.unlockReg(offset_reg_lock); + + const addr_reg, const addr_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(addr_lock); + try func.genSetReg(Type.u64, addr_reg, slice_mcv); + + _ = try func.addInst(.{ + .tag = .add, + .data = .{ .r_type = .{ + .rd = addr_reg, + .rs1 = addr_reg, + .rs2 = offset_reg, + } }, + }); + + return .{ .register = addr_reg }; +} + +fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const result_ty = func.typeOfIndex(inst); + + const array_ty = func.typeOf(bin_op.lhs); + const array_mcv = try func.resolveInst(bin_op.lhs); + + const index_mcv = try func.resolveInst(bin_op.rhs); + const index_ty = func.typeOf(bin_op.rhs); + + const elem_ty = array_ty.childType(zcu); + const elem_abi_size = elem_ty.abiSize(zcu); + + const addr_reg, const addr_reg_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(addr_reg_lock); + + switch (array_mcv) { + .register => { + const frame_index = try func.allocFrameIndex(FrameAlloc.initType(array_ty, zcu)); + try func.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv); + try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = .{ .index = frame_index } }); + }, + .load_frame => |frame_addr| { + try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = frame_addr }); + }, + else => try func.genSetReg(Type.u64, addr_reg, array_mcv.address()), + } + + const dst_mcv = try func.allocRegOrMem(result_ty, inst, false); + + if (array_ty.isVector(zcu)) { + // we need to load the vector, vslidedown to get the element we want + // and store that element in a load frame. + + const src_reg, const src_lock = try func.allocReg(.vector); + defer func.register_manager.unlockReg(src_lock); + + // load the vector into a temporary register + try func.genCopy(array_ty, .{ .register = src_reg }, .{ .indirect = .{ .reg = addr_reg } }); + + // we need to construct a 1xbitSize vector because of how lane splitting works in RISC-V + const single_ty = try pt.vectorType(.{ .child = elem_ty.toIntern(), .len = 1 }); + + // we can do a shortcut here where we don't need a vslicedown + // and can just copy to the frame index. + if (!(index_mcv == .immediate and index_mcv.immediate == 0)) { + const index_reg = try func.copyToTmpRegister(Type.u64, index_mcv); + + _ = try func.addInst(.{ + .tag = .vslidedownvx, + .data = .{ .r_type = .{ + .rd = src_reg, + .rs1 = index_reg, + .rs2 = src_reg, + } }, + }); + } + + try func.genCopy(single_ty, dst_mcv, .{ .register = src_reg }); + break :result dst_mcv; + } + + const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_abi_size); + const offset_lock = func.register_manager.lockRegAssumeUnused(offset_reg); + defer func.register_manager.unlockReg(offset_lock); + _ = try func.addInst(.{ + .tag = .add, + .data = .{ .r_type = .{ + .rd = addr_reg, + .rs1 = addr_reg, + .rs2 = offset_reg, + } }, + }); + + try func.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrElemVal(func: *Func, inst: Air.Inst.Index) !void { + const is_volatile = false; // TODO + const pt = func.pt; + const zcu = pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const base_ptr_ty = func.typeOf(bin_op.lhs); + + const result: MCValue = if (!is_volatile and func.liveness.isUnused(inst)) .unreach else result: { + const elem_ty = base_ptr_ty.elemType2(zcu); + if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const base_ptr_mcv = try func.resolveInst(bin_op.lhs); + const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); + + const index_mcv = try func.resolveInst(bin_op.rhs); + const index_lock: ?RegisterLock = switch (index_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_lock) |lock| func.register_manager.unlockReg(lock); + + const elem_ptr_reg, const elem_ptr_lock = if (base_ptr_mcv.isRegister() and + func.liveness.operandDies(inst, 0)) + .{ base_ptr_mcv.register, null } + else blk: { + const reg, const lock = try func.allocReg(.int); + try func.genSetReg(base_ptr_ty, reg, base_ptr_mcv); + break :blk .{ reg, lock }; + }; + defer if (elem_ptr_lock) |lock| func.register_manager.unlockReg(lock); + + try func.genBinOp( + .ptr_add, + base_ptr_mcv, + base_ptr_ty, + index_mcv, + Type.u64, + elem_ptr_reg, + ); + + const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + const dst_lock = switch (dst_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); + + try func.load(dst_mcv, .{ .register = elem_ptr_reg }, base_ptr_ty); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const elem_ptr_ty = func.typeOfIndex(inst); + const base_ptr_ty = func.typeOf(extra.lhs); + + if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { + @panic("audit"); + } + + const base_ptr_mcv = try func.resolveInst(extra.lhs); + const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); + + const index_mcv = try func.resolveInst(extra.rhs); + const index_lock: ?RegisterLock = switch (index_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_lock) |lock| func.register_manager.unlockReg(lock); + + const result_reg, const result_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(result_lock); + + try func.genBinOp( + .ptr_add, + base_ptr_mcv, + base_ptr_ty, + index_mcv, + Type.u64, + result_reg, + ); + + break :result MCValue{ .register = result_reg }; + }; + + return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airSetUnionTag(func: *Func, inst: Air.Inst.Index) !void { + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + _ = bin_op; + return func.fail("TODO implement airSetUnionTag for {}", .{func.target.cpu.arch}); + // return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const tag_ty = func.typeOfIndex(inst); + const union_ty = func.typeOf(ty_op.operand); + const layout = union_ty.unionGetLayout(zcu); + + if (layout.tag_size == 0) { + return func.finishAir(inst, .none, .{ ty_op.operand, .none, .none }); + } + + const operand = try func.resolveInst(ty_op.operand); + + const frame_mcv = try func.allocRegOrMem(union_ty, null, false); + try func.genCopy(union_ty, frame_mcv, operand); + + const tag_abi_size = tag_ty.abiSize(zcu); + const result_reg, const result_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(result_lock); + + switch (frame_mcv) { + .load_frame => { + if (tag_abi_size <= 8) { + const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align)) + @intCast(layout.payload_size) + else + 0; + + try func.genCopy( + tag_ty, + .{ .register = result_reg }, + frame_mcv.offset(off), + ); + } else { + return func.fail( + "TODO implement get_union_tag for ABI larger than 8 bytes and operand {}, tag {f}", + .{ frame_mcv, tag_ty.fmt(pt) }, + ); + } + }, + else => return func.fail("TODO: airGetUnionTag {s}", .{@tagName(operand)}), + } + + return func.finishAir(inst, .{ .register = result_reg }, .{ ty_op.operand, .none, .none }); +} + +fn airClz(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try func.resolveInst(ty_op.operand); + const ty = func.typeOf(ty_op.operand); + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const src_reg, const src_lock = try func.promoteReg(ty, operand); + defer if (src_lock) |lock| func.register_manager.unlockReg(lock); + + const dst_reg: Register = if (func.reuseOperand( + inst, + ty_op.operand, + 0, + operand, + ) and operand == .register) + operand.register + else + (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register; + + const bit_size = ty.bitSize(func.pt.zcu); + if (!math.isPowerOfTwo(bit_size)) try func.truncateRegister(ty, src_reg); + + if (bit_size > 64) { + return func.fail("TODO: airClz > 64 bits, found {d}", .{bit_size}); + } + + _ = try func.addInst(.{ + .tag = switch (bit_size) { + 32 => .clzw, + else => .clz, + }, + .data = .{ + .r_type = .{ + .rs2 = .zero, // rs2 is 0 filled in the spec + .rs1 = src_reg, + .rd = dst_reg, + }, + }, + }); + + if (!(bit_size == 32 or bit_size == 64)) { + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ .i_type = .{ + .rd = dst_reg, + .rs1 = dst_reg, + .imm12 = Immediate.s(-@as(i12, @intCast(64 - bit_size % 64))), + } }, + }); + } + + break :result .{ .register = dst_reg }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airCtz(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + _ = ty_op; + return func.fail("TODO: finish ctz", .{}); +} + +fn airPopcount(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const pt = func.pt; + const zcu = pt.zcu; + + const operand = try func.resolveInst(ty_op.operand); + const src_ty = func.typeOf(ty_op.operand); + const operand_reg, const operand_lock = try func.promoteReg(src_ty, operand); + defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); + + const dst_reg, const dst_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(dst_lock); + + const bit_size = src_ty.bitSize(zcu); + switch (bit_size) { + 32, 64 => {}, + 1...31, 33...63 => try func.truncateRegister(src_ty, operand_reg), + else => return func.fail("TODO: airPopcount > 64 bits", .{}), + } + + _ = try func.addInst(.{ + .tag = if (bit_size <= 32) .cpopw else .cpop, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = operand_reg, + .rs2 = @enumFromInt(0b00010), // this is the cpop funct5 + }, + }, + }); + + break :result .{ .register = dst_reg }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airAbs(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const ty = func.typeOf(ty_op.operand); + const scalar_ty = ty.scalarType(zcu); + const operand = try func.resolveInst(ty_op.operand); + + switch (scalar_ty.zigTypeTag(zcu)) { + .int => if (ty.zigTypeTag(zcu) == .vector) { + return func.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)}); + } else { + const int_info = scalar_ty.intInfo(zcu); + const int_bits = int_info.bits; + switch (int_bits) { + 32, 64 => {}, + else => return func.fail("TODO: airAbs Int size {d}", .{int_bits}), + } + + const return_mcv = try func.copyToNewRegister(inst, operand); + const operand_reg = return_mcv.register; + + const temp_reg, const temp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(temp_lock); + + _ = try func.addInst(.{ + .tag = switch (int_bits) { + 32 => .sraiw, + 64 => .srai, + else => unreachable, + }, + .data = .{ .i_type = .{ + .rd = temp_reg, + .rs1 = operand_reg, + .imm12 = Immediate.u(int_bits - 1), + } }, + }); + + _ = try func.addInst(.{ + .tag = .xor, + .data = .{ .r_type = .{ + .rd = operand_reg, + .rs1 = operand_reg, + .rs2 = temp_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = switch (int_bits) { + 32 => .subw, + 64 => .sub, + else => unreachable, + }, + .data = .{ .r_type = .{ + .rd = operand_reg, + .rs1 = operand_reg, + .rs2 = temp_reg, + } }, + }); + + break :result return_mcv; + }, + .float => { + const float_bits = scalar_ty.floatBits(zcu.getTarget()); + const mnem: Mnemonic = switch (float_bits) { + 16 => return func.fail("TODO: airAbs 16-bit float", .{}), + 32 => .fsgnjxs, + 64 => .fsgnjxd, + 80 => return func.fail("TODO: airAbs 80-bit float", .{}), + 128 => return func.fail("TODO: airAbs 128-bit float", .{}), + else => unreachable, + }; + + const return_mcv = try func.copyToNewRegister(inst, operand); + const operand_reg = return_mcv.register; + + assert(operand_reg.class() == .float); + + _ = try func.addInst(.{ + .tag = mnem, + .data = .{ + .r_type = .{ + .rd = operand_reg, + .rs1 = operand_reg, + .rs2 = operand_reg, + }, + }, + }); + + break :result return_mcv; + }, + else => return func.fail("TODO: implement airAbs {f}", .{scalar_ty.fmt(pt)}), + } + + break :result .unreach; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airByteSwap(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const pt = func.pt; + const zcu = pt.zcu; + const ty = func.typeOf(ty_op.operand); + const operand = try func.resolveInst(ty_op.operand); + + switch (ty.zigTypeTag(zcu)) { + .int => { + const int_bits = ty.intInfo(zcu).bits; + + // bytes are no-op + if (int_bits == 8 and func.reuseOperand(inst, ty_op.operand, 0, operand)) { + return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none }); + } + + const dest_mcv = try func.copyToNewRegister(inst, operand); + const dest_reg = dest_mcv.register; + + switch (int_bits) { + 16 => { + const temp_reg, const temp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(temp_lock); + + _ = try func.addInst(.{ + .tag = .srli, + .data = .{ .i_type = .{ + .imm12 = Immediate.s(8), + .rd = temp_reg, + .rs1 = dest_reg, + } }, + }); + + _ = try func.addInst(.{ + .tag = .slli, + .data = .{ .i_type = .{ + .imm12 = Immediate.s(8), + .rd = dest_reg, + .rs1 = dest_reg, + } }, + }); + _ = try func.addInst(.{ + .tag = .@"or", + .data = .{ .r_type = .{ + .rd = dest_reg, + .rs1 = dest_reg, + .rs2 = temp_reg, + } }, + }); + }, + else => return func.fail("TODO: {d} bits for airByteSwap", .{int_bits}), + } + + break :result dest_mcv; + }, + else => return func.fail("TODO: airByteSwap {f}", .{ty.fmt(pt)}), + } + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airBitReverse(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airBitReverse for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const pt = func.pt; + const zcu = pt.zcu; + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const ty = func.typeOf(un_op); + + const operand = try func.resolveInst(un_op); + const operand_bit_size = ty.bitSize(zcu); + + if (!math.isPowerOfTwo(operand_bit_size)) + return func.fail("TODO: airUnaryMath non-pow 2", .{}); + + const operand_reg, const operand_lock = try func.promoteReg(ty, operand); + defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); + + const dst_class = func.typeRegClass(ty); + const dst_reg, const dst_lock = try func.allocReg(dst_class); + defer func.register_manager.unlockReg(dst_lock); + + switch (ty.zigTypeTag(zcu)) { + .float => { + assert(dst_class == .float); + + switch (operand_bit_size) { + 16, 80, 128 => return func.fail("TODO: airUnaryMath Float bit-size {}", .{operand_bit_size}), + 32, 64 => {}, + else => unreachable, + } + + switch (tag) { + .sqrt => { + _ = try func.addInst(.{ + .tag = if (operand_bit_size == 64) .fsqrtd else .fsqrts, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = operand_reg, + .rs2 = .f0, // unused, spec says it's 0 + }, + }, + }); + }, + + else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}), + } + }, + .int => { + assert(dst_class == .int); + + switch (tag) { + else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}), + } + }, + else => return func.fail("TODO: airUnaryMath ty: {f}", .{ty.fmt(pt)}), + } + + break :result MCValue{ .register = dst_reg }; + }; + + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn reuseOperand( + func: *Func, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + op_index: Air.Liveness.OperandInt, + mcv: MCValue, +) bool { + return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); +} + +fn reuseOperandAdvanced( + func: *Func, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + op_index: Air.Liveness.OperandInt, + mcv: MCValue, + maybe_tracked_inst: ?Air.Inst.Index, +) bool { + if (!func.liveness.operandDies(inst, op_index)) + return false; + + switch (mcv) { + .register, + .register_pair, + => for (mcv.getRegs()) |reg| { + // If it's in the registers table, need to associate the register(s) with the + // new instruction. + if (maybe_tracked_inst) |tracked_inst| { + if (!func.register_manager.isRegFree(reg)) { + if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { + func.register_manager.registers[index] = tracked_inst; + } + } + } else func.register_manager.freeReg(reg); + }, + .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, + else => return false, + } + + // Prevent the operand deaths processing code from deallocating it. + func.reused_operands.set(op_index); + const op_inst = operand.toIndex().?; + func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst); + + return true; +} + +fn airLoad(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const elem_ty = func.typeOfIndex(inst); + + const result: MCValue = result: { + if (!elem_ty.hasRuntimeBits(zcu)) + break :result .none; + + const ptr = try func.resolveInst(ty_op.operand); + const is_volatile = func.typeOf(ty_op.operand).isVolatilePtr(zcu); + if (func.liveness.isUnused(inst) and !is_volatile) + break :result .unreach; + + const elem_size = elem_ty.abiSize(zcu); + + const dst_mcv: MCValue = blk: { + // The MCValue that holds the pointer can be re-used as the value. + // - "ptr" is 8 bytes, and if the element is more than that, we cannot reuse it. + // + // - "ptr" will be stored in an integer register, so the type that we're gonna + // load into it must also be a type that can be inside of an integer register + if (elem_size <= 8 and + (if (ptr == .register) func.typeRegClass(elem_ty) == ptr.register.class() else true) and + func.reuseOperand(inst, ty_op.operand, 0, ptr)) + { + break :blk ptr; + } else { + break :blk try func.allocRegOrMem(elem_ty, inst, true); + } + }; + + try func.load(dst_mcv, ptr, func.typeOf(ty_op.operand)); + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn load(func: *Func, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void { + const pt = func.pt; + const zcu = pt.zcu; + const dst_ty = ptr_ty.childType(zcu); + + log.debug("loading {}:{f} into {}", .{ ptr_mcv, ptr_ty.fmt(pt), dst_mcv }); + + switch (ptr_mcv) { + .none, + .undef, + .unreach, + .dead, + .register_pair, + .reserved_frame, + => unreachable, // not a valid pointer + + .immediate, + .register, + .register_offset, + .lea_frame, + .lea_symbol, + => try func.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()), + + .memory, + .indirect, + .load_symbol, + .load_frame, + => { + const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); + const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); + defer func.register_manager.unlockReg(addr_lock); + + try func.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); + }, + .air_ref => |ptr_ref| try func.load(dst_mcv, try func.resolveInst(ptr_ref), ptr_ty), + } +} + +fn airStore(func: *Func, inst: Air.Inst.Index, safety: bool) !void { + if (safety) { + // TODO if the value is undef, write 0xaa bytes to dest + } else { + // TODO if the value is undef, don't lower this instruction + } + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ptr = try func.resolveInst(bin_op.lhs); + const value = try func.resolveInst(bin_op.rhs); + const ptr_ty = func.typeOf(bin_op.lhs); + + try func.store(ptr, value, ptr_ty); + + return func.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +/// Loads `value` into the "payload" of `pointer`. +fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void { + const zcu = func.pt.zcu; + const src_ty = ptr_ty.childType(zcu); + log.debug("storing {}:{f} in {}:{f}", .{ src_mcv, src_ty.fmt(func.pt), ptr_mcv, ptr_ty.fmt(func.pt) }); + + switch (ptr_mcv) { + .none => unreachable, + .undef => unreachable, + .unreach => unreachable, + .dead => unreachable, + .register_pair => unreachable, + .reserved_frame => unreachable, + + .immediate, + .register, + .register_offset, + .lea_symbol, + .lea_frame, + => try func.genCopy(src_ty, ptr_mcv.deref(), src_mcv), + + .memory, + .indirect, + .load_symbol, + .load_frame, + => { + const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); + const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); + defer func.register_manager.unlockReg(addr_lock); + + try func.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); + }, + .air_ref => |ptr_ref| try func.store(try func.resolveInst(ptr_ref), src_mcv, ptr_ty), + } +} + +fn airStructFieldPtr(func: *Func, inst: Air.Inst.Index) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.StructField, ty_pl.payload).data; + const result = try func.structFieldPtr(inst, extra.struct_operand, extra.field_index); + return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); +} + +fn airStructFieldPtrIndex(func: *Func, inst: Air.Inst.Index, index: u8) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = try func.structFieldPtr(inst, ty_op.operand, index); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn structFieldPtr(func: *Func, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue { + const pt = func.pt; + const zcu = pt.zcu; + const ptr_field_ty = func.typeOfIndex(inst); + const ptr_container_ty = func.typeOf(operand); + const container_ty = ptr_container_ty.childType(zcu); + + const field_offset: i32 = switch (container_ty.containerLayout(zcu)) { + .auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)), + .@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) + + (if (zcu.typeToStruct(container_ty)) |struct_obj| zcu.structPackedFieldBitOffset(struct_obj, index) else 0) - + ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8), + }; + + const src_mcv = try func.resolveInst(operand); + const dst_mcv = if (switch (src_mcv) { + .immediate, .lea_frame => true, + .register, .register_offset => func.reuseOperand(inst, operand, 0, src_mcv), + else => false, + }) src_mcv else try func.copyToNewRegister(inst, src_mcv); + return dst_mcv.offset(field_offset); +} + +fn airStructFieldVal(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.StructField, ty_pl.payload).data; + const operand = extra.struct_operand; + const index = extra.field_index; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const src_mcv = try func.resolveInst(operand); + const struct_ty = func.typeOf(operand); + const field_ty = struct_ty.fieldType(index, zcu); + if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const field_off: u32 = switch (struct_ty.containerLayout(zcu)) { + .auto, .@"extern" => @intCast(struct_ty.structFieldOffset(index, zcu) * 8), + .@"packed" => if (zcu.typeToStruct(struct_ty)) |struct_type| + zcu.structPackedFieldBitOffset(struct_type, index) + else + 0, + }; + + switch (src_mcv) { + .dead, .unreach => unreachable, + .register => |src_reg| { + const src_reg_lock = func.register_manager.lockRegAssumeUnused(src_reg); + defer func.register_manager.unlockReg(src_reg_lock); + + const dst_reg = if (field_off == 0) + (try func.copyToNewRegister(inst, src_mcv)).register + else + try func.copyToTmpRegister(Type.u64, .{ .register = src_reg }); + + const dst_mcv: MCValue = .{ .register = dst_reg }; + const dst_lock = func.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); + + if (field_off > 0) { + _ = try func.addInst(.{ + .tag = .srli, + .data = .{ .i_type = .{ + .imm12 = Immediate.u(@intCast(field_off)), + .rd = dst_reg, + .rs1 = dst_reg, + } }, + }); + } + + if (field_off == 0) { + try func.truncateRegister(field_ty, dst_reg); + } + + break :result if (field_off == 0) dst_mcv else try func.copyToNewRegister(inst, dst_mcv); + }, + .load_frame => { + const field_abi_size: u32 = @intCast(field_ty.abiSize(zcu)); + if (field_off % 8 == 0) { + const field_byte_off = @divExact(field_off, 8); + const off_mcv = src_mcv.address().offset(@intCast(field_byte_off)).deref(); + const field_bit_size = field_ty.bitSize(zcu); + + if (field_abi_size <= 8) { + const int_ty = try pt.intType( + if (field_ty.isAbiInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned, + @intCast(field_bit_size), + ); + + const dst_reg, const dst_lock = try func.allocReg(.int); + const dst_mcv = MCValue{ .register = dst_reg }; + defer func.register_manager.unlockReg(dst_lock); + + try func.genCopy(int_ty, dst_mcv, off_mcv); + break :result try func.copyToNewRegister(inst, dst_mcv); + } + + const container_abi_size: u32 = @intCast(struct_ty.abiSize(zcu)); + const dst_mcv = if (field_byte_off + field_abi_size <= container_abi_size and + func.reuseOperand(inst, operand, 0, src_mcv)) + off_mcv + else dst: { + const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + try func.genCopy(field_ty, dst_mcv, off_mcv); + break :dst dst_mcv; + }; + if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) { + const tmp_reg, const tmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(tmp_lock); + + const hi_mcv = + dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref(); + try func.genSetReg(Type.u64, tmp_reg, hi_mcv); + try func.genCopy(Type.u64, hi_mcv, .{ .register = tmp_reg }); + } + break :result dst_mcv; + } + + return func.fail("TODO: airStructFieldVal load_frame field_off non multiple of 8", .{}); + }, + else => return func.fail("TODO: airStructField {s}", .{@tagName(src_mcv)}), + } + }; + + return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); +} + +fn airFieldParentPtr(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement codegen airFieldParentPtr", .{}); +} + +fn genArgDbgInfo(func: *const Func, name: []const u8, ty: Type, mcv: MCValue) InnerError!void { + assert(!func.mod.strip); + + // TODO: Add a pseudo-instruction or something to defer this work until Emit. + // We aren't allowed to interact with linker state here. + if (true) return; + switch (func.debug_output) { + .dwarf => |dw| switch (mcv) { + .register => |reg| dw.genLocalDebugInfo( + .local_arg, + name, + ty, + .{ .reg = reg.dwarfNum() }, + ) catch |err| return func.fail("failed to generate debug info: {s}", .{@errorName(err)}), + .load_frame => {}, + else => {}, + }, + .plan9 => {}, + .none => {}, + } +} + +fn airArg(func: *Func, inst: Air.Inst.Index) InnerError!void { + const zcu = func.pt.zcu; + + var arg_index = func.arg_index; + + // we skip over args that have no bits + while (func.args[arg_index] == .none) arg_index += 1; + func.arg_index = arg_index + 1; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const src_mcv = func.args[arg_index]; + const arg_ty = func.typeOfIndex(inst); + + const dst_mcv = try func.allocRegOrMem(arg_ty, inst, false); + + log.debug("airArg {} -> {}", .{ src_mcv, dst_mcv }); + + try func.genCopy(arg_ty, dst_mcv, src_mcv); + + const arg = func.air.instructions.items(.data)[@intFromEnum(inst)].arg; + // can delete `func.func_index` if this logic is moved to emit + const func_zir = zcu.funcInfo(func.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?; + const file = zcu.fileByIndex(func_zir.file); + const zir = &file.zir.?; + const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); + + try func.genArgDbgInfo(name, arg_ty, src_mcv); + break :result dst_mcv; + }; + + return func.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airTrap(func: *Func) !void { + _ = try func.addInst(.{ + .tag = .unimp, + .data = .none, + }); + return func.finishAirBookkeeping(); +} + +fn airBreakpoint(func: *Func) !void { + _ = try func.addInst(.{ + .tag = .ebreak, + .data = .none, + }); + return func.finishAirBookkeeping(); +} + +fn airRetAddr(func: *Func, inst: Air.Inst.Index) !void { + const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + try func.genCopy(Type.u64, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }); + return func.finishAir(inst, dst_mcv, .{ .none, .none, .none }); +} + +fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void { + const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + try func.genCopy(Type.u64, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }); + return func.finishAir(inst, dst_mcv, .{ .none, .none, .none }); +} + +fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void { + if (modifier == .always_tail) return func.fail("TODO implement tail calls for riscv64", .{}); + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const callee = pl_op.operand; + const extra = func.air.extraData(Air.Call, pl_op.payload); + const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]); + + const expected_num_args = 8; + const ExpectedContents = extern struct { + vals: [expected_num_args][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)), + }; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), func.gpa); + const allocator = stack.get(); + + const arg_tys = try allocator.alloc(Type, arg_refs.len); + defer allocator.free(arg_tys); + for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = func.typeOf(arg_ref); + + const arg_vals = try allocator.alloc(MCValue, arg_refs.len); + defer allocator.free(arg_vals); + for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref }; + + const call_ret = try func.genCall(.{ .air = callee }, arg_tys, arg_vals); + + var bt = func.liveness.iterateBigTomb(inst); + try func.feed(&bt, pl_op.operand); + for (arg_refs) |arg_ref| try func.feed(&bt, arg_ref); + + const result = if (func.liveness.isUnused(inst)) .unreach else call_ret; + return func.finishAirResult(inst, result); +} + +fn genCall( + func: *Func, + info: union(enum) { + air: Air.Inst.Ref, + lib: struct { + return_type: InternPool.Index, + param_types: []const InternPool.Index, + lib: ?[]const u8 = null, + callee: []const u8, + }, + }, + arg_tys: []const Type, + args: []const MCValue, +) !MCValue { + const pt = func.pt; + const zcu = pt.zcu; + + const fn_ty = switch (info) { + .air => |callee| fn_info: { + const callee_ty = func.typeOf(callee); + break :fn_info switch (callee_ty.zigTypeTag(zcu)) { + .@"fn" => callee_ty, + .pointer => callee_ty.childType(zcu), + else => unreachable, + }; + }, + .lib => |lib| try pt.funcType(.{ + .param_types = lib.param_types, + .return_type = lib.return_type, + .cc = func.target.cCallingConvention().?, + }), + }; + + const fn_info = zcu.typeToFunc(fn_ty).?; + + const allocator = func.gpa; + + const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len); + defer allocator.free(var_args); + for (var_args, arg_tys[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; + + var call_info = try func.resolveCallingConventionValues(fn_info, var_args); + defer call_info.deinit(func); + + // We need a properly aligned and sized call frame to be able to call this function. + { + const needed_call_frame = FrameAlloc.init(.{ + .size = call_info.stack_byte_count, + .alignment = call_info.stack_align, + }); + const frame_allocs_slice = func.frame_allocs.slice(); + const stack_frame_size = + &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size); + const stack_frame_align = + &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align); + } + + var reg_locks = std.array_list.Managed(?RegisterLock).init(allocator); + defer reg_locks.deinit(); + try reg_locks.ensureTotalCapacity(8); + defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| func.register_manager.unlockReg(lock); + + const frame_indices = try allocator.alloc(FrameIndex, args.len); + defer allocator.free(frame_indices); + + switch (call_info.return_value.long) { + .none, .unreach => {}, + .indirect => |reg_off| try func.register_manager.getReg(reg_off.reg, null), + else => unreachable, + } + for (call_info.args, args, arg_tys, frame_indices) |dst_arg, src_arg, arg_ty, *frame_index| { + switch (dst_arg) { + .none => {}, + .register => |reg| { + try func.register_manager.getReg(reg, null); + try reg_locks.append(func.register_manager.lockReg(reg)); + }, + .register_pair => |regs| { + for (regs) |reg| try func.register_manager.getReg(reg, null); + try reg_locks.appendSlice(&func.register_manager.lockRegs(2, regs)); + }, + .indirect => |reg_off| { + frame_index.* = try func.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu)); + try func.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg); + try func.register_manager.getReg(reg_off.reg, null); + try reg_locks.append(func.register_manager.lockReg(reg_off.reg)); + }, + else => return func.fail("TODO: genCall set arg {s}", .{@tagName(dst_arg)}), + } + } + + switch (call_info.return_value.long) { + .none, .unreach => {}, + .indirect => |reg_off| { + const ret_ty = Type.fromInterned(fn_info.return_type); + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu)); + try func.genSetReg(Type.u64, reg_off.reg, .{ + .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, + }); + call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } }; + try reg_locks.append(func.register_manager.lockReg(reg_off.reg)); + }, + else => unreachable, + } + + for (call_info.args, arg_tys, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| { + switch (dst_arg) { + .none, .load_frame => {}, + .register_pair => try func.genCopy(arg_ty, dst_arg, src_arg), + .register => |dst_reg| try func.genSetReg( + arg_ty, + dst_reg, + src_arg, + ), + .indirect => |reg_off| try func.genSetReg(Type.u64, reg_off.reg, .{ + .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, + }), + else => return func.fail("TODO: genCall actual set {s}", .{@tagName(dst_arg)}), + } + } + + // Due to incremental compilation, how function calls are generated depends + // on linking. + switch (info) { + .air => |callee| { + if (try func.air.value(callee, pt)) |func_value| { + const func_key = zcu.intern_pool.indexToKey(func_value.ip_index); + switch (switch (func_key) { + else => func_key, + .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { + .nav => |nav| zcu.intern_pool.indexToKey(zcu.navValue(nav).toIntern()), + else => func_key, + } else func_key, + }) { + .func => |func_val| { + if (func.bin_file.cast(.elf)) |elf_file| { + const zo = elf_file.zigObjectPtr().?; + const sym_index = try zo.getOrCreateMetadataForNav(zcu, func_val.owner_nav); + + if (func.mod.pic) { + return func.fail("TODO: genCall pic", .{}); + } else { + try func.genSetReg(Type.u64, .ra, .{ .lea_symbol = .{ .sym = sym_index } }); + _ = try func.addInst(.{ + .tag = .jalr, + .data = .{ .i_type = .{ + .rd = .ra, + .rs1 = .ra, + .imm12 = Immediate.s(0), + } }, + }); + } + } else unreachable; // not a valid riscv64 format + }, + .@"extern" => |@"extern"| { + const lib_name = @"extern".lib_name.toSlice(&zcu.intern_pool); + const name = @"extern".name.toSlice(&zcu.intern_pool); + const atom_index = try func.owner.getSymbolIndex(func); + + const elf_file = func.bin_file.cast(.elf).?; + _ = try func.addInst(.{ + .tag = .pseudo_extern_fn_reloc, + .data = .{ .reloc = .{ + .register = .ra, + .atom_index = atom_index, + .sym_index = try elf_file.getGlobalSymbol(name, lib_name), + } }, + }); + }, + else => return func.fail("TODO implement calling bitcasted functions", .{}), + } + } else { + assert(func.typeOf(callee).zigTypeTag(zcu) == .pointer); + const addr_reg, const addr_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(addr_lock); + try func.genSetReg(Type.u64, addr_reg, .{ .air_ref = callee }); + + _ = try func.addInst(.{ + .tag = .jalr, + .data = .{ .i_type = .{ + .rd = .ra, + .rs1 = addr_reg, + .imm12 = Immediate.s(0), + } }, + }); + } + }, + .lib => return func.fail("TODO: lib func calls", .{}), + } + + // reset the vector settings as they might have changed in the function + func.avl = null; + func.vtype = null; + + return call_info.return_value.short; +} + +fn airRet(func: *Func, inst: Air.Inst.Index, safety: bool) !void { + const pt = func.pt; + const zcu = pt.zcu; + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + + if (safety) { + // safe + } else { + // not safe + } + + const ret_ty = func.fn_type.fnReturnType(zcu); + switch (func.ret_mcv.short) { + .none => {}, + .register, + .register_pair, + => { + if (ret_ty.isVector(zcu)) { + const bit_size = ret_ty.totalVectorBits(zcu); + + // set the vtype to hold the entire vector's contents in a single element + try func.setVl(.zero, 0, .{ + .vsew = switch (bit_size) { + 8 => .@"8", + 16 => .@"16", + 32 => .@"32", + 64 => .@"64", + else => unreachable, + }, + .vlmul = .m1, + .vma = true, + .vta = true, + }); + } + + try func.genCopy(ret_ty, func.ret_mcv.short, .{ .air_ref = un_op }); + }, + .indirect => |reg_off| { + try func.register_manager.getReg(reg_off.reg, null); + const lock = func.register_manager.lockRegAssumeUnused(reg_off.reg); + defer func.register_manager.unlockReg(lock); + + try func.genSetReg(Type.u64, reg_off.reg, func.ret_mcv.long); + try func.genSetMem( + .{ .reg = reg_off.reg }, + reg_off.off, + ret_ty, + .{ .air_ref = un_op }, + ); + }, + else => unreachable, + } + + func.ret_mcv.liveOut(func, inst); + try func.finishAir(inst, .unreach, .{ un_op, .none, .none }); + + // Just add space for an instruction, reloced this later + const index = try func.addInst(.{ + .tag = .pseudo_j, + .data = .{ .j_type = .{ + .rd = .zero, + .inst = undefined, + } }, + }); + + try func.exitlude_jump_relocs.append(func.gpa, index); +} + +fn airRetLoad(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ptr = try func.resolveInst(un_op); + + const ptr_ty = func.typeOf(un_op); + switch (func.ret_mcv.short) { + .none => {}, + .register, .register_pair => try func.load(func.ret_mcv.short, ptr, ptr_ty), + .indirect => |reg_off| try func.genSetReg(ptr_ty, reg_off.reg, ptr), + else => unreachable, + } + func.ret_mcv.liveOut(func, inst); + try func.finishAir(inst, .unreach, .{ un_op, .none, .none }); + + // Just add space for an instruction, reloced this later + const index = try func.addInst(.{ + .tag = .pseudo_j, + .data = .{ .j_type = .{ + .rd = .zero, + .inst = undefined, + } }, + }); + + try func.exitlude_jump_relocs.append(func.gpa, index); +} + +fn airCmp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const pt = func.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const lhs_ty = func.typeOf(bin_op.lhs); + + switch (lhs_ty.zigTypeTag(zcu)) { + .int, + .@"enum", + .bool, + .pointer, + .error_set, + .optional, + .@"struct", + => { + const int_ty = switch (lhs_ty.zigTypeTag(zcu)) { + .@"enum" => lhs_ty.intTagType(zcu), + .int => lhs_ty, + .bool => Type.u1, + .pointer => Type.u64, + .error_set => Type.anyerror, + .optional => blk: { + const payload_ty = lhs_ty.optionalChild(zcu); + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :blk Type.u1; + } else if (lhs_ty.isPtrLikeOptional(zcu)) { + break :blk Type.u64; + } else { + return func.fail("TODO riscv cmp non-pointer optionals", .{}); + } + }, + .@"struct" => blk: { + const struct_obj = ip.loadStructType(lhs_ty.toIntern()); + assert(struct_obj.layout == .@"packed"); + const backing_index = struct_obj.backingIntTypeUnordered(ip); + break :blk Type.fromInterned(backing_index); + }, + else => unreachable, + }; + + const int_info = int_ty.intInfo(zcu); + if (int_info.bits <= 64) { + break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); + } else { + return func.fail("TODO riscv cmp for ints > 64 bits", .{}); + } + }, + .float => { + const float_bits = lhs_ty.floatBits(func.target); + const float_reg_size: u32 = if (func.hasFeature(.d)) 64 else 32; + if (float_bits > float_reg_size) { + return func.fail("TODO: airCmp float > 64/32 bits", .{}); + } + break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs); + }, + else => unreachable, + } + }; + + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airCmpVector(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement airCmpVector for {}", .{func.target.cpu.arch}); +} + +fn airCmpLtErrorsLen(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try func.resolveInst(un_op); + _ = operand; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLtErrorsLen for {}", .{func.target.cpu.arch}); + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void { + const dbg_stmt = func.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; + + _ = try func.addInst(.{ + .tag = .pseudo_dbg_line_column, + .data = .{ .pseudo_dbg_line_column = .{ + .line = dbg_stmt.line, + .column = dbg_stmt.column, + } }, + }); + + return func.finishAirBookkeeping(); +} + +fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len])); +} + +fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void { + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const operand = pl_op.operand; + const ty = func.typeOf(operand); + const mcv = try func.resolveInst(operand); + const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); + + const tag = func.air.instructions.items(.tag)[@intFromEnum(inst)]; + func.genVarDbgInfo(tag, ty, mcv, name.toSlice(func.air)) catch |err| + return func.fail("failed to generate variable debug info: {s}", .{@errorName(err)}); + + return func.finishAir(inst, .unreach, .{ operand, .none, .none }); +} + +fn genVarDbgInfo( + func: Func, + tag: Air.Inst.Tag, + ty: Type, + mcv: MCValue, + name: []const u8, +) !void { + // TODO: Add a pseudo-instruction or something to defer this work until Emit. + // We aren't allowed to interact with linker state here. + if (true) return; + switch (func.debug_output) { + .dwarf => |dwarf| { + const loc: link.File.Dwarf.Loc = switch (mcv) { + .register => |reg| .{ .reg = reg.dwarfNum() }, + .memory => |address| .{ .constu = address }, + .immediate => |x| .{ .constu = x }, + .none => .empty, + else => blk: { + // log.warn("TODO generate debug info for {}", .{mcv}); + break :blk .empty; + }, + }; + try dwarf.genLocalDebugInfo(switch (tag) { + else => unreachable, + .dbg_var_ptr, .dbg_var_val => .local_var, + .dbg_arg_inline => .local_arg, + }, name, ty, loc); + }, + .plan9 => {}, + .none => {}, + } +} + +fn airCondBr(func: *Func, inst: Air.Inst.Index) !void { + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const cond = try func.resolveInst(pl_op.operand); + const cond_ty = func.typeOf(pl_op.operand); + const extra = func.air.extraData(Air.CondBr, pl_op.payload); + const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const liveness_cond_br = func.liveness.getCondBr(inst); + + // If the condition dies here in this condbr instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + if (func.liveness.operandDies(inst, 0)) { + if (pl_op.operand.toIndex()) |op_inst| try func.processDeath(op_inst); + } + + func.scope_generation += 1; + const state = try func.saveState(); + const reloc = try func.condBr(cond_ty, cond); + + for (liveness_cond_br.then_deaths) |death| try func.processDeath(death); + try func.genBody(then_body); + try func.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + func.performReloc(reloc); + + for (liveness_cond_br.else_deaths) |death| try func.processDeath(death); + try func.genBody(else_body); + try func.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + // We already took care of pl_op.operand earlier, so there's nothing left to do. + func.finishAirBookkeeping(); +} + +fn condBr(func: *Func, cond_ty: Type, condition: MCValue) !Mir.Inst.Index { + const cond_reg = try func.copyToTmpRegister(cond_ty, condition); + + return try func.addInst(.{ + .tag = .beq, + .data = .{ + .b_type = .{ + .rs1 = cond_reg, + .rs2 = .zero, + .inst = undefined, + }, + }, + }); +} + +fn isNull(func: *Func, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue { + const pt = func.pt; + const zcu = pt.zcu; + const pl_ty = opt_ty.optionalChild(zcu); + + const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu)) + .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty } + else + .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool }; + + const return_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + assert(return_mcv == .register); // should not be larger 8 bytes + const return_reg = return_mcv.register; + + switch (opt_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .register_offset, + .lea_frame, + .lea_symbol, + .reserved_frame, + .air_ref, + .register_pair, + => unreachable, + + .register => |opt_reg| { + if (some_info.off == 0) { + _ = try func.addInst(.{ + .tag = .pseudo_compare, + .data = .{ + .compare = .{ + .op = .eq, + .rd = return_reg, + .rs1 = opt_reg, + .rs2 = try func.copyToTmpRegister( + some_info.ty, + .{ .immediate = 0 }, + ), + .ty = Type.bool, + }, + }, + }); + return return_mcv; + } + assert(some_info.ty.ip_index == .bool_type); + const bit_offset: u7 = @intCast(some_info.off * 8); + + try func.genBinOp( + .shr, + .{ .register = opt_reg }, + Type.u64, + .{ .immediate = bit_offset }, + Type.u8, + return_reg, + ); + try func.truncateRegister(Type.u8, return_reg); + try func.genBinOp( + .cmp_eq, + .{ .register = return_reg }, + Type.u64, + .{ .immediate = 0 }, + Type.u8, + return_reg, + ); + + return return_mcv; + }, + + .load_frame => { + const opt_reg = try func.copyToTmpRegister( + some_info.ty, + opt_mcv.address().offset(some_info.off).deref(), + ); + const opt_reg_lock = func.register_manager.lockRegAssumeUnused(opt_reg); + defer func.register_manager.unlockReg(opt_reg_lock); + + _ = try func.addInst(.{ + .tag = .pseudo_compare, + .data = .{ + .compare = .{ + .op = .eq, + .rd = return_reg, + .rs1 = opt_reg, + .rs2 = try func.copyToTmpRegister( + some_info.ty, + .{ .immediate = 0 }, + ), + .ty = Type.bool, + }, + }, + }); + return return_mcv; + }, + + else => return func.fail("TODO: isNull {}", .{opt_mcv}), + } +} + +fn airIsNull(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try func.resolveInst(un_op); + const ty = func.typeOf(un_op); + const result = try func.isNull(inst, ty, operand); + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airIsNullPtr(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try func.resolveInst(un_op); + _ = operand; + const ty = func.typeOf(un_op); + _ = ty; + + if (true) return func.fail("TODO: airIsNullPtr", .{}); + + return func.finishAir(inst, .unreach, .{ un_op, .none, .none }); +} + +fn airIsNonNull(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try func.resolveInst(un_op); + const ty = func.typeOf(un_op); + const result = try func.isNull(inst, ty, operand); + assert(result == .register); + + _ = try func.addInst(.{ + .tag = .pseudo_not, + .data = .{ + .rr = .{ + .rd = result.register, + .rs = result.register, + }, + }, + }); + + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airIsNonNullPtr(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try func.resolveInst(un_op); + _ = operand; + const ty = func.typeOf(un_op); + _ = ty; + + if (true) return func.fail("TODO: airIsNonNullPtr", .{}); + + return func.finishAir(inst, .unreach, .{ un_op, .none, .none }); +} + +fn airIsErr(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const operand = try func.resolveInst(un_op); + const operand_ty = func.typeOf(un_op); + break :result try func.isErr(inst, operand_ty, operand); + }; + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airIsErrPtr(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const operand_ptr = try func.resolveInst(un_op); + const operand: MCValue = blk: { + if (func.reuseOperand(inst, un_op, 0, operand_ptr)) { + // The MCValue that holds the pointer can be re-used as the value. + break :blk operand_ptr; + } else { + break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + } + }; + try func.load(operand, operand_ptr, func.typeOf(un_op)); + const operand_ptr_ty = func.typeOf(un_op); + const operand_ty = operand_ptr_ty.childType(zcu); + + break :result try func.isErr(inst, operand_ty, operand); + }; + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +/// Generates a compare instruction which will indicate if `eu_mcv` is an error. +/// +/// Result is in the return register. +fn isErr(func: *Func, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue { + _ = maybe_inst; + const zcu = func.pt.zcu; + const err_ty = eu_ty.errorUnionSet(zcu); + if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false + const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); + + const return_reg, const return_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(return_lock); + + switch (eu_mcv) { + .register => |reg| { + const eu_lock = func.register_manager.lockReg(reg); + defer if (eu_lock) |lock| func.register_manager.unlockReg(lock); + + try func.genCopy(eu_ty, .{ .register = return_reg }, eu_mcv); + + if (err_off > 0) { + try func.genBinOp( + .shr, + .{ .register = return_reg }, + eu_ty, + .{ .immediate = @as(u6, @intCast(err_off * 8)) }, + Type.u8, + return_reg, + ); + } + + try func.genBinOp( + .cmp_neq, + .{ .register = return_reg }, + Type.anyerror, + .{ .immediate = 0 }, + Type.u8, + return_reg, + ); + }, + .load_frame => |frame_addr| { + try func.genBinOp( + .cmp_neq, + .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + err_off, + } }, + Type.anyerror, + .{ .immediate = 0 }, + Type.anyerror, + return_reg, + ); + }, + else => return func.fail("TODO implement isErr for {}", .{eu_mcv}), + } + + return .{ .register = return_reg }; +} + +fn airIsNonErr(func: *Func, inst: Air.Inst.Index) !void { + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const operand = try func.resolveInst(un_op); + const ty = func.typeOf(un_op); + break :result try func.isNonErr(inst, ty, operand); + }; + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn isNonErr(func: *Func, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue { + const is_err_res = try func.isErr(inst, eu_ty, eu_mcv); + switch (is_err_res) { + .register => |reg| { + _ = try func.addInst(.{ + .tag = .pseudo_not, + .data = .{ + .rr = .{ + .rd = reg, + .rs = reg, + }, + }, + }); + return is_err_res; + }, + // always false case + .immediate => |imm| { + assert(imm == 0); + return MCValue{ .immediate = @intFromBool(imm == 0) }; + }, + else => unreachable, + } +} + +fn airIsNonErrPtr(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const operand_ptr = try func.resolveInst(un_op); + const operand: MCValue = blk: { + if (func.reuseOperand(inst, un_op, 0, operand_ptr)) { + // The MCValue that holds the pointer can be re-used as the value. + break :blk operand_ptr; + } else { + break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); + } + }; + const operand_ptr_ty = func.typeOf(un_op); + const operand_ty = operand_ptr_ty.childType(zcu); + + try func.load(operand, operand_ptr, func.typeOf(un_op)); + break :result try func.isNonErr(inst, operand_ty, operand); + }; + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airLoop(func: *Func, inst: Air.Inst.Index) !void { + // A loop is a setup to be able to jump back to the beginning. + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const loop = func.air.extraData(Air.Block, ty_pl.payload); + const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]); + + func.scope_generation += 1; + const state = try func.saveState(); + + try func.loops.putNoClobber(func.gpa, inst, .{ + .state = state, + .jmp_target = @intCast(func.mir_instructions.len), + }); + defer assert(func.loops.remove(inst)); + + try func.genBody(body); + + func.finishAirBookkeeping(); +} + +/// Send control flow to the `index` of `func.code`. +fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index { + return func.addInst(.{ + .tag = .pseudo_j, + .data = .{ .j_type = .{ + .rd = .zero, + .inst = index, + } }, + }); +} + +fn airBlock(func: *Func, inst: Air.Inst.Index) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Block, ty_pl.payload); + try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len])); +} + +fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { + // A block is a setup to be able to jump to the end. + const inst_tracking_i = func.inst_tracking.count(); + func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach)); + + func.scope_generation += 1; + try func.blocks.putNoClobber(func.gpa, inst, .{ .state = func.initRetroactiveState() }); + const liveness = func.liveness.getBlock(inst); + + // TODO emit debug info lexical block + try func.genBody(body); + + var block_data = func.blocks.fetchRemove(inst).?; + defer block_data.value.deinit(func.gpa); + if (block_data.value.relocs.items.len > 0) { + try func.restoreState(block_data.value.state, liveness.deaths, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + for (block_data.value.relocs.items) |reloc| func.performReloc(reloc); + } + + if (std.debug.runtime_safety) assert(func.inst_tracking.getIndex(inst).? == inst_tracking_i); + const tracking = &func.inst_tracking.values()[inst_tracking_i]; + if (func.liveness.isUnused(inst)) try tracking.die(func, inst); + func.getValueIfFree(tracking.short, inst); + func.finishAirBookkeeping(); +} + +fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { + const switch_br = func.air.unwrapSwitch(inst); + const condition = try func.resolveInst(switch_br.operand); + + // If the condition dies here in this switch instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + if (func.liveness.operandDies(inst, 0)) { + if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); + } + + try func.lowerSwitchBr(inst, switch_br, condition); + + // We already took care of pl_op.operand earlier, so there's nothing left to do + func.finishAirBookkeeping(); +} + +fn lowerSwitchBr( + func: *Func, + inst: Air.Inst.Index, + switch_br: Air.UnwrappedSwitch, + condition: MCValue, +) !void { + const condition_ty = func.typeOf(switch_br.operand); + const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1); + defer func.gpa.free(liveness.deaths); + + func.scope_generation += 1; + const state = try func.saveState(); + + var it = switch_br.iterateCases(); + while (it.next()) |case| { + var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); + defer func.gpa.free(relocs); + + for (case.items, relocs[0..case.items.len]) |item, *reloc| { + const item_mcv = try func.resolveInst(item); + + const cond_lock = switch (condition) { + .register => func.register_manager.lockRegAssumeUnused(condition.register), + else => null, + }; + defer if (cond_lock) |lock| func.register_manager.unlockReg(lock); + + const cmp_reg, const cmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(cmp_lock); + + try func.genBinOp( + .cmp_neq, + condition, + condition_ty, + item_mcv, + condition_ty, + cmp_reg, + ); + + reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg }); + } + + for (case.ranges, relocs[case.items.len..]) |range, *reloc| { + const min_mcv = try func.resolveInst(range[0]); + const max_mcv = try func.resolveInst(range[1]); + const cond_lock = switch (condition) { + .register => func.register_manager.lockRegAssumeUnused(condition.register), + else => null, + }; + defer if (cond_lock) |lock| func.register_manager.unlockReg(lock); + + const temp_cmp_reg, const temp_cmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(temp_cmp_lock); + + // is `condition` less than `min`? is "true", we've failed + try func.genBinOp( + .cmp_gte, + condition, + condition_ty, + min_mcv, + condition_ty, + temp_cmp_reg, + ); + + // if the compare was true, we will jump to the fail case and fall through + // to the next checks + const lt_fail_reloc = try func.condBr(condition_ty, .{ .register = temp_cmp_reg }); + try func.genBinOp( + .cmp_gt, + condition, + condition_ty, + max_mcv, + condition_ty, + temp_cmp_reg, + ); + + reloc.* = try func.condBr(condition_ty, .{ .register = temp_cmp_reg }); + func.performReloc(lt_fail_reloc); + } + + const skip_case_reloc = try func.jump(undefined); + + for (liveness.deaths[case.idx]) |operand| try func.processDeath(operand); + + for (relocs) |reloc| func.performReloc(reloc); + try func.genBody(case.body); + try func.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + func.performReloc(skip_case_reloc); + } + + if (switch_br.else_body_len > 0) { + const else_body = it.elseBody(); + + const else_deaths = liveness.deaths.len - 1; + for (liveness.deaths[else_deaths]) |operand| try func.processDeath(operand); + + try func.genBody(else_body); + try func.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + } +} + +fn airLoopSwitchBr(func: *Func, inst: Air.Inst.Index) !void { + const switch_br = func.air.unwrapSwitch(inst); + const condition = try func.resolveInst(switch_br.operand); + + const mat_cond = if (condition.isMutable() and + func.reuseOperand(inst, switch_br.operand, 0, condition)) + condition + else mat_cond: { + const ty = func.typeOf(switch_br.operand); + const mat_cond = try func.allocRegOrMem(ty, inst, true); + try func.genCopy(ty, mat_cond, condition); + break :mat_cond mat_cond; + }; + func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond)); + + // If the condition dies here in this switch instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + if (func.liveness.operandDies(inst, 0)) { + if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); + } + + func.scope_generation += 1; + const state = try func.saveState(); + + try func.loops.putNoClobber(func.gpa, inst, .{ + .state = state, + .jmp_target = @intCast(func.mir_instructions.len), + }); + defer assert(func.loops.remove(inst)); + + // Stop tracking block result without forgetting tracking info + try func.freeValue(mat_cond); + + try func.lowerSwitchBr(inst, switch_br, mat_cond); + + try func.processDeath(inst); + func.finishAirBookkeeping(); +} + +fn airSwitchDispatch(func: *Func, inst: Air.Inst.Index) !void { + const br = func.air.instructions.items(.data)[@intFromEnum(inst)].br; + + const block_ty = func.typeOfIndex(br.block_inst); + const block_tracking = func.inst_tracking.getPtr(br.block_inst).?; + const loop_data = func.loops.getPtr(br.block_inst).?; + done: { + try func.getValue(block_tracking.short, null); + const src_mcv = try func.resolveInst(br.operand); + + if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { + try func.getValue(block_tracking.short, br.block_inst); + // .long = .none to avoid merging operand and block result stack frames. + const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; + try current_tracking.materializeUnsafe(func, br.block_inst, block_tracking.*); + for (current_tracking.getRegs()) |src_reg| func.register_manager.freeReg(src_reg); + break :done; + } + + try func.getValue(block_tracking.short, br.block_inst); + const dst_mcv = block_tracking.short; + try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand)); + break :done; + } + + // Process operand death so that it is properly accounted for in the State below. + if (func.liveness.operandDies(inst, 0)) { + if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); + } + + try func.restoreState(loop_data.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + + // Emit a jump with a relocation. It will be patched up after the block ends. + // Leave the jump offset undefined + _ = try func.jump(loop_data.jmp_target); + + // Stop tracking block result without forgetting tracking info + try func.freeValue(block_tracking.short); + + func.finishAirBookkeeping(); +} + +fn performReloc(func: *Func, inst: Mir.Inst.Index) void { + const tag = func.mir_instructions.items(.tag)[inst]; + const target: Mir.Inst.Index = @intCast(func.mir_instructions.len); + + switch (tag) { + .beq, + .bne, + => func.mir_instructions.items(.data)[inst].b_type.inst = target, + .jal => func.mir_instructions.items(.data)[inst].j_type.inst = target, + .pseudo_j => func.mir_instructions.items(.data)[inst].j_type.inst = target, + else => std.debug.panic("TODO: performReloc {s}", .{@tagName(tag)}), + } +} + +fn airBr(func: *Func, inst: Air.Inst.Index) !void { + const zcu = func.pt.zcu; + const br = func.air.instructions.items(.data)[@intFromEnum(inst)].br; + + const block_ty = func.typeOfIndex(br.block_inst); + const block_unused = + !block_ty.hasRuntimeBitsIgnoreComptime(zcu) or func.liveness.isUnused(br.block_inst); + const block_tracking = func.inst_tracking.getPtr(br.block_inst).?; + const block_data = func.blocks.getPtr(br.block_inst).?; + const first_br = block_data.relocs.items.len == 0; + const block_result = result: { + if (block_unused) break :result .none; + + if (!first_br) try func.getValue(block_tracking.short, null); + const src_mcv = try func.resolveInst(br.operand); + + if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { + if (first_br) break :result src_mcv; + + try func.getValue(block_tracking.short, br.block_inst); + try InstTracking.materializeUnsafe( + // .long = .none to avoid merging operand and block result stack frames. + .{ .long = .none, .short = src_mcv }, + func, + br.block_inst, + block_tracking.*, + ); + try func.freeValue(src_mcv); + break :result block_tracking.short; + } + + const dst_mcv = if (first_br) try func.allocRegOrMem(block_ty, br.block_inst, true) else dst: { + try func.getValue(block_tracking.short, br.block_inst); + break :dst block_tracking.short; + }; + try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand)); + break :result dst_mcv; + }; + + // Process operand death so that it is properly accounted for in the State below. + if (func.liveness.operandDies(inst, 0)) { + if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); + } + + if (first_br) { + block_tracking.* = InstTracking.init(block_result); + try func.saveRetroactiveState(&block_data.state); + } else try func.restoreState(block_data.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + + // Emit a jump with a relocation. It will be patched up after the block ends. + // Leave the jump offset undefined + const jmp_reloc = try func.jump(undefined); + try block_data.relocs.append(func.gpa, jmp_reloc); + + // Stop tracking block result without forgetting tracking info + try func.freeValue(block_tracking.short); + + func.finishAirBookkeeping(); +} + +fn airRepeat(func: *Func, inst: Air.Inst.Index) !void { + const loop_inst = func.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst; + const repeat_info = func.loops.get(loop_inst).?; + try func.restoreState(repeat_info.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = true, + }); + _ = try func.jump(repeat_info.jmp_target); + func.finishAirBookkeeping(); +} + +fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void { + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const tag: Air.Inst.Tag = func.air.instructions.items(.tag)[@intFromEnum(inst)]; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const lhs = try func.resolveInst(bin_op.lhs); + const rhs = try func.resolveInst(bin_op.rhs); + const lhs_ty = Type.bool; + const rhs_ty = Type.bool; + + const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs); + defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock); + + const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs); + defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock); + + const result_reg, const result_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(result_lock); + + _ = try func.addInst(.{ + .tag = if (tag == .bool_or) .@"or" else .@"and", + .data = .{ .r_type = .{ + .rd = result_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + } }, + }); + + // safety truncate + if (func.wantSafety()) { + _ = try func.addInst(.{ + .tag = .andi, + .data = .{ .i_type = .{ + .rd = result_reg, + .rs1 = result_reg, + .imm12 = Immediate.s(1), + } }, + }); + } + + break :result .{ .register = result_reg }; + }; + return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airAsm(func: *Func, inst: Air.Inst.Index) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Asm, ty_pl.payload); + const outputs_len = extra.data.flags.outputs_len; + var extra_i: usize = extra.end; + const outputs: []const Air.Inst.Ref = + @ptrCast(func.air.extra.items[extra_i..][0..outputs_len]); + extra_i += outputs.len; + const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]); + extra_i += inputs.len; + + var result: MCValue = .none; + var args = std.array_list.Managed(MCValue).init(func.gpa); + try args.ensureTotalCapacity(outputs.len + inputs.len); + defer { + for (args.items) |arg| if (arg.getReg()) |reg| func.register_manager.unlockReg(.{ + .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue, + }); + args.deinit(); + } + var arg_map = std.StringHashMap(u8).init(func.gpa); + try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len)); + defer arg_map.deinit(); + + var outputs_extra_i = extra_i; + for (outputs) |output| { + const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]); + const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0); + const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + const is_read = switch (constraint[0]) { + '=' => false, + '+' => read: { + if (output == .none) return func.fail( + "read-write constraint unsupported for asm result: '{s}'", + .{constraint}, + ); + break :read true; + }, + else => return func.fail("invalid constraint: '{s}'", .{constraint}), + }; + const is_early_clobber = constraint[1] == '&'; + const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..]; + const arg_mcv: MCValue = arg_mcv: { + const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "m")) + if (output != .none) null else return func.fail( + "memory constraint unsupported for asm result: '{s}'", + .{constraint}, + ) + else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}")) + parseRegName(rest["{".len .. rest.len - "}".len]) orelse + return func.fail("invalid register constraint: '{s}'", .{constraint}) + else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { + const index = std.fmt.charToDigit(rest[0], 10) catch unreachable; + if (index >= args.items.len) return func.fail("constraint out of bounds: '{s}'", .{ + constraint, + }); + break :arg_mcv args.items[index]; + } else return func.fail("invalid constraint: '{s}'", .{constraint}); + break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { + const ptr_mcv = try func.resolveInst(output); + switch (ptr_mcv) { + .immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| + break :arg ptr_mcv.deref(), + .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(), + else => {}, + } + break :arg .{ .indirect = .{ .reg = try func.copyToTmpRegister(Type.usize, ptr_mcv) } }; + }; + }; + if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { + _ = func.register_manager.lockReg(reg); + }; + if (!mem.eql(u8, name, "_")) + arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); + args.appendAssumeCapacity(arg_mcv); + if (output == .none) result = arg_mcv; + if (is_read) try func.load(arg_mcv, .{ .air_ref = output }, func.typeOf(output)); + } + + for (inputs) |input| { + const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]); + const constraint = mem.sliceTo(input_bytes, 0); + const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + const ty = func.typeOf(input); + const input_mcv = try func.resolveInst(input); + const arg_mcv: MCValue = if (mem.eql(u8, constraint, "X")) + input_mcv + else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: { + const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse + return func.fail("invalid register constraint: '{s}'", .{constraint}); + try func.register_manager.getReg(reg, null); + try func.genSetReg(ty, reg, input_mcv); + break :arg .{ .register = reg }; + } else if (mem.eql(u8, constraint, "r")) arg: { + switch (input_mcv) { + .register => break :arg input_mcv, + else => {}, + } + const temp_reg = try func.copyToTmpRegister(ty, input_mcv); + break :arg .{ .register = temp_reg }; + } else return func.fail("invalid input constraint: '{s}'", .{constraint}); + if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { + _ = func.register_manager.lockReg(reg); + }; + if (!mem.eql(u8, name, "_")) + arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); + args.appendAssumeCapacity(arg_mcv); + } + + const zcu = func.pt.zcu; + const ip = &zcu.intern_pool; + const aggregate = ip.indexToKey(extra.data.clobbers).aggregate; + const struct_type: Type = .fromInterned(aggregate.ty); + switch (aggregate.storage) { + .elems => |elems| for (elems, 0..) |elem, i| { + switch (elem) { + .bool_true => { + const clobber = struct_type.structFieldName(i, zcu).toSlice(ip).?; + assert(clobber.len != 0); + if (std.mem.eql(u8, clobber, "memory")) { + // nothing really to do + } else { + try func.register_manager.getReg(parseRegName(clobber) orelse + return func.fail("invalid clobber: '{s}'", .{clobber}), null); + } + }, + .bool_false => continue, + else => unreachable, + } + }, + .repeated_elem => |elem| switch (elem) { + .bool_true => @panic("TODO"), + .bool_false => {}, + else => unreachable, + }, + .bytes => @panic("TODO"), + } + + const Label = struct { + target: Mir.Inst.Index = undefined, + pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + + const Kind = enum { definition, reference }; + + fn isValid(kind: Kind, name: []const u8) bool { + for (name, 0..) |c, i| switch (c) { + else => return false, + '$' => if (i == 0) return false, + '.' => {}, + '0'...'9' => if (i == 0) switch (kind) { + .definition => if (name.len != 1) return false, + .reference => { + if (name.len != 2) return false; + switch (name[1]) { + else => return false, + 'B', 'F', 'b', 'f' => {}, + } + }, + }, + '@', 'A'...'Z', '_', 'a'...'z' => {}, + }; + return name.len > 0; + } + }; + var labels: std.StringHashMapUnmanaged(Label) = .empty; + defer { + var label_it = labels.valueIterator(); + while (label_it.next()) |label| label.pending_relocs.deinit(func.gpa); + labels.deinit(func.gpa); + } + + const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len]; + var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;"); + next_line: while (line_it.next()) |line| { + var mnem_it = mem.tokenizeAny(u8, line, " \t"); + const mnem_str = while (mnem_it.next()) |mnem_str| { + if (mem.startsWith(u8, mnem_str, "#")) continue :next_line; + if (mem.startsWith(u8, mnem_str, "//")) continue :next_line; + if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str; + const label_name = mnem_str[0 .. mnem_str.len - ":".len]; + if (!Label.isValid(.definition, label_name)) + return func.fail("invalid label: '{s}'", .{label_name}); + + const label_gop = try labels.getOrPut(func.gpa, label_name); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else { + const anon = std.ascii.isDigit(label_name[0]); + if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0) + return func.fail("redefined label: '{s}'", .{label_name}); + for (label_gop.value_ptr.pending_relocs.items) |pending_reloc| + func.performReloc(pending_reloc); + if (anon) + label_gop.value_ptr.pending_relocs.clearRetainingCapacity() + else + label_gop.value_ptr.pending_relocs.clearAndFree(func.gpa); + } + label_gop.value_ptr.target = @intCast(func.mir_instructions.len); + } else continue; + + const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } = + if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem| + .{ .mnem = mnem } + else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo| + .{ .pseudo = pseudo } + else + return func.fail("invalid mnem str '{s}'", .{mnem_str}); + + const Operand = union(enum) { + none, + reg: Register, + imm: Immediate, + inst: Mir.Inst.Index, + sym: SymbolOffset, + }; + + var ops: [4]Operand = .{.none} ** 4; + var last_op = false; + var op_it = mem.splitAny(u8, mnem_it.rest(), ",("); + next_op: for (&ops) |*op| { + const op_str = while (!last_op) { + const full_str = op_it.next() orelse break :next_op; + const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse + mem.indexOf(u8, full_str, "//")) |comment| + code: { + last_op = true; + break :code full_str[0..comment]; + } else full_str; + const trim_str = mem.trim(u8, code_str, " \t*"); + if (trim_str.len > 0) break trim_str; + } else break; + + if (parseRegName(op_str)) |reg| { + op.* = .{ .reg = reg }; + } else if (std.fmt.parseInt(i12, op_str, 10)) |int| { + op.* = .{ .imm = Immediate.s(int) }; + } else |_| if (mem.startsWith(u8, op_str, "%[")) { + const mod_index = mem.indexOf(u8, op_str, "]@"); + const modifier = if (mod_index) |index| + op_str[index + "]@".len ..] + else + ""; + + op.* = switch (args.items[ + arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse + return func.fail("no matching constraint: '{s}'", .{op_str}) + ]) { + .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { + assert(sym_off.off == 0); + break :blk .{ .sym = sym_off }; + } else return func.fail("invalid modifier: '{s}'", .{modifier}), + .register => |reg| if (modifier.len == 0) + .{ .reg = reg } + else + return func.fail("invalid modified '{s}'", .{modifier}), + else => return func.fail("invalid constraint: '{s}'", .{op_str}), + }; + } else if (mem.endsWith(u8, op_str, ")")) { + const reg = op_str[0 .. op_str.len - ")".len]; + const addr_reg = parseRegName(reg) orelse + return func.fail("expected valid register, found '{s}'", .{reg}); + + op.* = .{ .reg = addr_reg }; + } else if (Label.isValid(.reference, op_str)) { + const anon = std.ascii.isDigit(op_str[0]); + const label_gop = try labels.getOrPut(func.gpa, op_str[0..if (anon) 1 else op_str.len]); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; + if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) + return func.fail("undefined label: '{s}'", .{op_str}); + const pending_relocs = &label_gop.value_ptr.pending_relocs; + if (if (anon) + op_str[1] == 'f' or op_str[1] == 'F' + else + !label_gop.found_existing or pending_relocs.items.len > 0) + try pending_relocs.append(func.gpa, @intCast(func.mir_instructions.len)); + op.* = .{ .inst = label_gop.value_ptr.target }; + } else return func.fail("invalid operand: '{s}'", .{op_str}); + } else if (op_it.next()) |op_str| return func.fail("extra operand: '{s}'", .{op_str}); + + switch (instruction) { + .mnem => |mnem| { + _ = (switch (ops[0]) { + .none => try func.addInst(.{ + .tag = mnem, + .data = .none, + }), + .reg => |reg1| switch (ops[1]) { + .reg => |reg2| switch (ops[2]) { + .imm => |imm1| try func.addInst(.{ + .tag = mnem, + .data = .{ .i_type = .{ + .rd = reg1, + .rs1 = reg2, + .imm12 = imm1, + } }, + }), + else => error.InvalidInstruction, + }, + .imm => |imm1| switch (ops[2]) { + .reg => |reg2| switch (mnem) { + .sd => try func.addInst(.{ + .tag = mnem, + .data = .{ .i_type = .{ + .rd = reg2, + .rs1 = reg1, + .imm12 = imm1, + } }, + }), + .ld => try func.addInst(.{ + .tag = mnem, + .data = .{ .i_type = .{ + .rd = reg1, + .rs1 = reg2, + .imm12 = imm1, + } }, + }), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .none => switch (mnem) { + .jalr => try func.addInst(.{ + .tag = mnem, + .data = .{ .i_type = .{ + .rd = .ra, + .rs1 = reg1, + .imm12 = Immediate.s(0), + } }, + }), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }) catch |err| { + switch (err) { + error.InvalidInstruction => return func.fail( + "invalid instruction: {s} {s} {s} {s} {s}", + .{ + @tagName(mnem), + @tagName(ops[0]), + @tagName(ops[1]), + @tagName(ops[2]), + @tagName(ops[3]), + }, + ), + else => |e| return e, + } + }; + }, + .pseudo => |pseudo| { + (@as(error{InvalidInstruction}!void, switch (pseudo) { + .li => blk: { + if (ops[0] != .reg or ops[1] != .imm) { + break :blk error.InvalidInstruction; + } + + const reg = ops[0].reg; + const imm = ops[1].imm; + + try func.genSetReg(Type.usize, reg, .{ .immediate = imm.asBits(u64) }); + }, + .mv => blk: { + if (ops[0] != .reg or ops[1] != .reg) { + break :blk error.InvalidInstruction; + } + + const dst = ops[0].reg; + const src = ops[1].reg; + + if (dst.class() != .int or src.class() != .int) { + return func.fail("pseudo instruction 'mv' only works on integer registers", .{}); + } + + try func.genSetReg(Type.usize, dst, .{ .register = src }); + }, + .tail => blk: { + if (ops[0] != .sym) { + break :blk error.InvalidInstruction; + } + + const sym_offset = ops[0].sym; + assert(sym_offset.off == 0); + + const random_link_reg, const lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(lock); + + _ = try func.addInst(.{ + .tag = .pseudo_extern_fn_reloc, + .data = .{ .reloc = .{ + .register = random_link_reg, + .atom_index = try func.owner.getSymbolIndex(func), + .sym_index = sym_offset.sym, + } }, + }); + }, + .ret => _ = try func.addInst(.{ + .tag = .jalr, + .data = .{ .i_type = .{ + .rd = .zero, + .rs1 = .ra, + .imm12 = Immediate.s(0), + } }, + }), + .beqz => blk: { + if (ops[0] != .reg or ops[1] != .inst) { + break :blk error.InvalidInstruction; + } + + _ = try func.addInst(.{ + .tag = .beq, + .data = .{ .b_type = .{ + .rs1 = ops[0].reg, + .rs2 = .zero, + .inst = ops[1].inst, + } }, + }); + }, + })) catch |err| { + switch (err) { + error.InvalidInstruction => return func.fail( + "invalid instruction: {s} {s} {s} {s} {s}", + .{ + @tagName(pseudo), + @tagName(ops[0]), + @tagName(ops[1]), + @tagName(ops[2]), + @tagName(ops[3]), + }, + ), + else => |e| return e, + } + }; + }, + } + } + + var label_it = labels.iterator(); + while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0) + return func.fail("undefined label: '{s}'", .{label.key_ptr.*}); + + for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { + const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]); + const constraint = + mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0); + const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + if (output == .none) continue; + if (arg_mcv != .register) continue; + if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue; + try func.store(.{ .air_ref = output }, arg_mcv, func.typeOf(output)); + } + + simple: { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); + var buf_index: usize = 0; + for (outputs) |output| { + if (output == .none) continue; + + if (buf_index >= buf.len) break :simple; + buf[buf_index] = output; + buf_index += 1; + } + if (buf_index + inputs.len > buf.len) break :simple; + @memcpy(buf[buf_index..][0..inputs.len], inputs); + return func.finishAir(inst, result, buf); + } + var bt = func.liveness.iterateBigTomb(inst); + for (outputs) |output| if (output != .none) try func.feed(&bt, output); + for (inputs) |input| try func.feed(&bt, input); + return func.finishAirResult(inst, result); +} + +/// Sets the value of `dst_mcv` to the value of `src_mcv`. +fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { + // There isn't anything to store + if (dst_mcv == .none) return; + + if (!dst_mcv.isMutable()) { + // panic so we can see the trace + return std.debug.panic("tried to genCopy immutable: {s}", .{@tagName(dst_mcv)}); + } + + const zcu = func.pt.zcu; + + switch (dst_mcv) { + .register => |reg| return func.genSetReg(ty, reg, src_mcv), + .register_offset => |dst_reg_off| try func.genSetReg(ty, dst_reg_off.reg, switch (src_mcv) { + .none, + .unreach, + .dead, + .undef, + => unreachable, + .immediate, + .register, + .register_offset, + => src_mcv.offset(-dst_reg_off.off), + else => .{ .register_offset = .{ + .reg = try func.copyToTmpRegister(ty, src_mcv), + .off = -dst_reg_off.off, + } }, + }), + .indirect => |reg_off| try func.genSetMem( + .{ .reg = reg_off.reg }, + reg_off.off, + ty, + src_mcv, + ), + .load_frame => |frame_addr| try func.genSetMem( + .{ .frame = frame_addr.index }, + frame_addr.off, + ty, + src_mcv, + ), + .load_symbol => { + const addr_reg, const addr_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(addr_lock); + + try func.genSetReg(ty, addr_reg, dst_mcv.address()); + try func.genCopy(ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); + }, + .memory => return func.fail("TODO: genCopy memory", .{}), + .register_pair => |dst_regs| { + const src_info: ?struct { addr_reg: Register, addr_lock: ?RegisterLock } = switch (src_mcv) { + .register_pair, .memory, .indirect, .load_frame => null, + .load_symbol => src: { + const src_addr_reg, const src_addr_lock = try func.promoteReg(Type.u64, src_mcv.address()); + errdefer func.register_manager.unlockReg(src_addr_lock); + + break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; + }, + .air_ref => |src_ref| return func.genCopy( + ty, + dst_mcv, + try func.resolveInst(src_ref), + ), + else => return func.fail("genCopy register_pair src: {}", .{src_mcv}), + }; + + defer if (src_info) |info| { + if (info.addr_lock) |lock| { + func.register_manager.unlockReg(lock); + } + }; + + var part_disp: i32 = 0; + for (dst_regs, try func.splitType(ty), 0..) |dst_reg, dst_ty, part_i| { + try func.genSetReg(dst_ty, dst_reg, switch (src_mcv) { + .register_pair => |src_regs| .{ .register = src_regs[part_i] }, + .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), + .load_symbol => .{ .indirect = .{ + .reg = src_info.?.addr_reg, + .off = part_disp, + } }, + else => unreachable, + }); + part_disp += @intCast(dst_ty.abiSize(zcu)); + } + }, + else => return std.debug.panic("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }), + } +} + +fn genInlineMemcpy( + func: *Func, + dst_ptr: MCValue, + src_ptr: MCValue, + len: MCValue, +) !void { + const regs = try func.register_manager.allocRegs(4, .{null} ** 4, abi.Registers.Integer.temporary); + const locks = func.register_manager.lockRegsAssumeUnused(4, regs); + defer for (locks) |lock| func.register_manager.unlockReg(lock); + + const count = regs[0]; + const tmp = regs[1]; + const src = regs[2]; + const dst = regs[3]; + + try func.genSetReg(Type.u64, count, len); + try func.genSetReg(Type.u64, src, src_ptr); + try func.genSetReg(Type.u64, dst, dst_ptr); + + // if count is 0, there's nothing to copy + _ = try func.addInst(.{ + .tag = .beq, + .data = .{ .b_type = .{ + .rs1 = count, + .rs2 = .zero, + .inst = @intCast(func.mir_instructions.len + 9), + } }, + }); + + // lb tmp, 0(src) + const first_inst = try func.addInst(.{ + .tag = .lb, + .data = .{ + .i_type = .{ + .rd = tmp, + .rs1 = src, + .imm12 = Immediate.s(0), + }, + }, + }); + + // sb tmp, 0(dst) + _ = try func.addInst(.{ + .tag = .sb, + .data = .{ + .i_type = .{ + .rd = dst, + .rs1 = tmp, + .imm12 = Immediate.s(0), + }, + }, + }); + + // dec count by 1 + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ + .i_type = .{ + .rd = count, + .rs1 = count, + .imm12 = Immediate.s(-1), + }, + }, + }); + + // branch if count is 0 + _ = try func.addInst(.{ + .tag = .beq, + .data = .{ + .b_type = .{ + .inst = @intCast(func.mir_instructions.len + 4), // points after the last inst + .rs1 = count, + .rs2 = .zero, + }, + }, + }); + + // increment the pointers + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ + .i_type = .{ + .rd = src, + .rs1 = src, + .imm12 = Immediate.s(1), + }, + }, + }); + + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ + .i_type = .{ + .rd = dst, + .rs1 = dst, + .imm12 = Immediate.s(1), + }, + }, + }); + + // jump back to start of loop + _ = try func.addInst(.{ + .tag = .pseudo_j, + .data = .{ .j_type = .{ + .rd = .zero, + .inst = first_inst, + } }, + }); +} + +fn genInlineMemset( + func: *Func, + dst_ptr: MCValue, + src_value: MCValue, + len: MCValue, +) !void { + const regs = try func.register_manager.allocRegs(3, .{null} ** 3, abi.Registers.Integer.temporary); + const locks = func.register_manager.lockRegsAssumeUnused(3, regs); + defer for (locks) |lock| func.register_manager.unlockReg(lock); + + const count = regs[0]; + const src = regs[1]; + const dst = regs[2]; + + try func.genSetReg(Type.u64, count, len); + try func.genSetReg(Type.u64, src, src_value); + try func.genSetReg(Type.u64, dst, dst_ptr); + + // sb src, 0(dst) + const first_inst = try func.addInst(.{ + .tag = .sb, + .data = .{ + .i_type = .{ + .rd = dst, + .rs1 = src, + .imm12 = Immediate.s(0), + }, + }, + }); + + // dec count by 1 + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ + .i_type = .{ + .rd = count, + .rs1 = count, + .imm12 = Immediate.s(-1), + }, + }, + }); + + // branch if count is 0 + _ = try func.addInst(.{ + .tag = .beq, + .data = .{ + .b_type = .{ + .inst = @intCast(func.mir_instructions.len + 3), // points after the last inst + .rs1 = count, + .rs2 = .zero, + }, + }, + }); + + // increment the pointers + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ + .i_type = .{ + .rd = dst, + .rs1 = dst, + .imm12 = Immediate.s(1), + }, + }, + }); + + // jump back to start of loop + _ = try func.addInst(.{ + .tag = .pseudo_j, + .data = .{ .j_type = .{ + .rd = .zero, + .inst = first_inst, + } }, + }); +} + +/// Sets the value of `src_mcv` into `reg`. Assumes you have a lock on it. +fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!void { + const pt = func.pt; + const zcu = pt.zcu; + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + + const max_size: u32 = switch (reg.class()) { + .int => 64, + .float => if (func.hasFeature(.d)) 64 else 32, + .vector => 64, // TODO: calculate it from avl * vsew + }; + if (abi_size > max_size) return std.debug.panic("tried to set reg with size {}", .{abi_size}); + const dst_reg_class = reg.class(); + + switch (src_mcv) { + .unreach, + .none, + .dead, + => unreachable, + .undef => |sym_index| { + if (!func.wantSafety()) + return; + + if (sym_index) |index| { + return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } }); + } + + switch (abi_size) { + 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }), + 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }), + 3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }), + 5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }), + else => unreachable, + } + }, + .immediate => |unsigned_x| { + assert(dst_reg_class == .int); + + const x: i64 = @bitCast(unsigned_x); + if (math.minInt(i12) <= x and x <= math.maxInt(i12)) { + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ .i_type = .{ + .rd = reg, + .rs1 = .zero, + .imm12 = Immediate.s(@intCast(x)), + } }, + }); + } else if (math.minInt(i32) <= x and x <= math.maxInt(i32)) { + const lo12: i12 = @truncate(x); + const carry: i32 = if (lo12 < 0) 1 else 0; + const hi20: i20 = @truncate((x >> 12) +% carry); + + _ = try func.addInst(.{ + .tag = .lui, + .data = .{ .u_type = .{ + .rd = reg, + .imm20 = Immediate.s(hi20), + } }, + }); + _ = try func.addInst(.{ + .tag = .addi, + .data = .{ .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.s(lo12), + } }, + }); + } else { + // TODO: use a more advanced myriad seq to do this without a reg. + // see: https://github.com/llvm/llvm-project/blob/081a66ffacfe85a37ff775addafcf3371e967328/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp#L224 + + const temp, const temp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(temp_lock); + + const lo32: i32 = @truncate(x); + const carry: i32 = if (lo32 < 0) 1 else 0; + const hi32: i32 = @truncate((x >> 32) +% carry); + + try func.genSetReg(Type.i32, temp, .{ .immediate = @bitCast(@as(i64, lo32)) }); + try func.genSetReg(Type.i32, reg, .{ .immediate = @bitCast(@as(i64, hi32)) }); + + _ = try func.addInst(.{ + .tag = .slli, + .data = .{ .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(32), + } }, + }); + + _ = try func.addInst(.{ + .tag = .add, + .data = .{ .r_type = .{ + .rd = reg, + .rs1 = reg, + .rs2 = temp, + } }, + }); + } + }, + .register => |src_reg| { + // If the registers are the same, nothing to do. + if (src_reg.id() == reg.id()) + return; + + // there is no instruction for loading the contents of a vector register + // into an integer register, however we can cheat a bit by setting the element + // size to the total size of the vector, and vmv.x.s will work then + if (src_reg.class() == .vector) { + try func.setVl(.zero, 0, .{ + .vsew = switch (ty.totalVectorBits(zcu)) { + 8 => .@"8", + 16 => .@"16", + 32 => .@"32", + 64 => .@"64", + else => |vec_bits| return func.fail("TODO: genSetReg vec -> {s} bits {d}", .{ + @tagName(reg.class()), + vec_bits, + }), + }, + .vlmul = .m1, + .vta = true, + .vma = true, + }); + } + + // mv reg, src_reg + _ = try func.addInst(.{ + .tag = .pseudo_mv, + .data = .{ .rr = .{ + .rd = reg, + .rs = src_reg, + } }, + }); + }, + // useful in cases like slice_ptr, which can easily reuse the operand + // but we need to get only the pointer out. + .register_pair => |pair| try func.genSetReg(ty, reg, .{ .register = pair[0] }), + .load_frame => |frame| { + if (reg.class() == .vector) { + // vectors don't support an offset memory load so we need to put the true + // address into a register before loading from it. + const addr_reg, const addr_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(addr_lock); + + try func.genCopy(ty, .{ .register = addr_reg }, src_mcv.address()); + try func.genCopy(ty, .{ .register = reg }, .{ .indirect = .{ .reg = addr_reg } }); + } else { + _ = try func.addInst(.{ + .tag = .pseudo_load_rm, + .data = .{ .rm = .{ + .r = reg, + .m = .{ + .base = .{ .frame = frame.index }, + .mod = .{ + .size = func.memSize(ty), + .unsigned = ty.isUnsignedInt(zcu), + .disp = frame.off, + }, + }, + } }, + }); + } + }, + .memory => |addr| { + try func.genSetReg(ty, reg, .{ .immediate = addr }); + + _ = try func.addInst(.{ + .tag = .ld, + .data = .{ .i_type = .{ + .rd = reg, + .rs1 = reg, + .imm12 = Immediate.u(0), + } }, + }); + }, + .lea_frame, .register_offset => { + _ = try func.addInst(.{ + .tag = .pseudo_lea_rm, + .data = .{ + .rm = .{ + .r = reg, + .m = switch (src_mcv) { + .register_offset => |reg_off| .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ + .size = .byte, // the size doesn't matter + .disp = reg_off.off, + .unsigned = false, + }, + }, + .lea_frame => |frame| .{ + .base = .{ .frame = frame.index }, + .mod = .{ + .size = .byte, // the size doesn't matter + .disp = frame.off, + .unsigned = false, + }, + }, + else => unreachable, + }, + }, + }, + }); + }, + .indirect => |reg_off| { + const load_tag: Mnemonic = switch (reg.class()) { + .float => switch (abi_size) { + 1 => unreachable, // Zig does not support 8-bit floats + 2 => return func.fail("TODO: genSetReg indirect 16-bit float", .{}), + 4 => .flw, + 8 => .fld, + else => return std.debug.panic("TODO: genSetReg for float size {d}", .{abi_size}), + }, + .int => switch (abi_size) { + 1...1 => .lb, + 2...2 => .lh, + 3...4 => .lw, + 5...8 => .ld, + else => return std.debug.panic("TODO: genSetReg for int size {d}", .{abi_size}), + }, + .vector => { + assert(reg_off.off == 0); + + // There is no vector instruction for loading with an offset to a base register, + // so we need to get an offset register containing the address of the vector first + // and load from it. + const len = ty.vectorLen(zcu); + const elem_ty = ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + try func.setVl(.zero, len, .{ + .vsew = switch (elem_size) { + 1 => .@"8", + 2 => .@"16", + 4 => .@"32", + 8 => .@"64", + else => unreachable, + }, + .vlmul = .m1, + .vma = true, + .vta = true, + }); + + _ = try func.addInst(.{ + .tag = .pseudo_load_rm, + .data = .{ .rm = .{ + .r = reg, + .m = .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ + .size = func.memSize(elem_ty), + .unsigned = false, + .disp = 0, + }, + }, + } }, + }); + + return; + }, + }; + + _ = try func.addInst(.{ + .tag = load_tag, + .data = .{ .i_type = .{ + .rd = reg, + .rs1 = reg_off.reg, + .imm12 = Immediate.s(reg_off.off), + } }, + }); + }, + .lea_symbol => |sym_off| { + assert(sym_off.off == 0); + const atom_index = try func.owner.getSymbolIndex(func); + + _ = try func.addInst(.{ + .tag = .pseudo_load_symbol, + .data = .{ .reloc = .{ + .register = reg, + .atom_index = atom_index, + .sym_index = sym_off.sym, + } }, + }); + }, + .load_symbol => { + const addr_reg, const addr_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(addr_lock); + + try func.genSetReg(ty, addr_reg, src_mcv.address()); + try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } }); + }, + .air_ref => |ref| try func.genSetReg(ty, reg, try func.resolveInst(ref)), + else => return func.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}), + } +} + +fn genSetMem( + func: *Func, + base: Memory.Base, + disp: i32, + ty: Type, + src_mcv: MCValue, +) InnerError!void { + const pt = func.pt; + const zcu = pt.zcu; + + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const dst_ptr_mcv: MCValue = switch (base) { + .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, + .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, + }; + switch (src_mcv) { + .none, + .unreach, + .dead, + .reserved_frame, + => unreachable, + .undef => |sym_index| { + if (sym_index) |index| { + return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } }); + } + + try func.genInlineMemset( + dst_ptr_mcv, + src_mcv, + .{ .immediate = abi_size }, + ); + }, + .register_offset, + .memory, + .indirect, + .load_frame, + .lea_frame, + .load_symbol, + .lea_symbol, + => switch (abi_size) { + 0 => {}, + 1, 2, 4, 8 => { + const reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary); + const src_lock = func.register_manager.lockRegAssumeUnused(reg); + defer func.register_manager.unlockReg(src_lock); + + try func.genSetReg(ty, reg, src_mcv); + try func.genSetMem(base, disp, ty, .{ .register = reg }); + }, + else => try func.genInlineMemcpy( + dst_ptr_mcv, + src_mcv.address(), + .{ .immediate = abi_size }, + ), + }, + .register => |reg| { + if (reg.class() == .vector) { + const addr_reg = try func.copyToTmpRegister(Type.u64, dst_ptr_mcv); + + const num_elem = ty.vectorLen(zcu); + const elem_size = ty.childType(zcu).bitSize(zcu); + + try func.setVl(.zero, num_elem, .{ + .vsew = switch (elem_size) { + 8 => .@"8", + 16 => .@"16", + 32 => .@"32", + 64 => .@"64", + else => unreachable, + }, + .vlmul = .m1, + .vma = true, + .vta = true, + }); + + _ = try func.addInst(.{ + .tag = .pseudo_store_rm, + .data = .{ .rm = .{ + .r = reg, + .m = .{ + .base = .{ .reg = addr_reg }, + .mod = .{ + .disp = 0, + .size = func.memSize(ty.childType(zcu)), + .unsigned = false, + }, + }, + } }, + }); + + return; + } + + const mem_size = switch (base) { + .frame => |base_fi| mem_size: { + assert(disp >= 0); + const frame_abi_size = func.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)]; + const frame_spill_pad = func.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)]; + assert(frame_abi_size - frame_spill_pad - disp >= abi_size); + break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size) + frame_abi_size + else + abi_size; + }, + else => abi_size, + }; + const src_size = math.ceilPowerOfTwoAssert(u32, abi_size); + const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size)); + if (src_size > mem_size) { + const frame_index = try func.allocFrameIndex(FrameAlloc.init(.{ + .size = src_size, + .alignment = src_align, + })); + const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } }; + _ = try func.addInst(.{ + .tag = .pseudo_store_rm, + .data = .{ .rm = .{ + .r = reg, + .m = .{ + .base = .{ .frame = frame_index }, + .mod = .{ + .size = Memory.Size.fromByteSize(src_size), + .unsigned = false, + }, + }, + } }, + }); + try func.genSetMem(base, disp, ty, frame_mcv); + try func.freeValue(frame_mcv); + } else _ = try func.addInst(.{ + .tag = .pseudo_store_rm, + .data = .{ .rm = .{ + .r = reg, + .m = .{ + .base = base, + .mod = .{ + .size = func.memSize(ty), + .disp = disp, + .unsigned = false, + }, + }, + } }, + }); + }, + .register_pair => |src_regs| { + var part_disp: i32 = disp; + for (try func.splitType(ty), src_regs) |src_ty, src_reg| { + try func.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }); + part_disp += @intCast(src_ty.abiSize(zcu)); + } + }, + .immediate => { + // TODO: remove this lock in favor of a copyToTmpRegister when we load 64 bit immediates with + // a register allocation. + const reg, const reg_lock = try func.promoteReg(ty, src_mcv); + defer if (reg_lock) |lock| func.register_manager.unlockReg(lock); + + return func.genSetMem(base, disp, ty, .{ .register = reg }); + }, + .air_ref => |src_ref| try func.genSetMem(base, disp, ty, try func.resolveInst(src_ref)), + } +} + +fn airBitCast(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = if (func.liveness.isUnused(inst)) .unreach else result: { + const src_mcv = try func.resolveInst(ty_op.operand); + + const src_ty = func.typeOf(ty_op.operand); + if (src_ty.toIntern() == .bool_type) break :result src_mcv; + const dst_ty = func.typeOfIndex(inst); + + const src_lock = if (src_mcv.getReg()) |reg| func.register_manager.lockReg(reg) else null; + defer if (src_lock) |lock| func.register_manager.unlockReg(lock); + + const dst_mcv = if (dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and src_mcv != .register_pair and + func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { + const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true); + try func.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { + .lt => dst_ty, + .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, + .gt => src_ty, + }, dst_mcv, src_mcv); + break :dst dst_mcv; + }; + + if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and + dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv; + + const abi_size = dst_ty.abiSize(zcu); + const bit_size = dst_ty.bitSize(zcu); + if (abi_size * 8 <= bit_size) break :result dst_mcv; + + return func.fail("TODO: airBitCast {f} to {f}", .{ src_ty.fmt(pt), dst_ty.fmt(pt) }); + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airArrayToSlice(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const slice_ty = func.typeOfIndex(inst); + const ptr_ty = func.typeOf(ty_op.operand); + const ptr = try func.resolveInst(ty_op.operand); + const array_ty = ptr_ty.childType(zcu); + const array_len = array_ty.arrayLen(zcu); + + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu)); + try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr); + try func.genSetMem( + .{ .frame = frame_index }, + @intCast(ptr_ty.abiSize(zcu)), + Type.u64, + .{ .immediate = array_len }, + ); + + const result = MCValue{ .load_frame = .{ .index = frame_index } }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airFloatFromInt(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const pt = func.pt; + const zcu = pt.zcu; + + const operand = try func.resolveInst(ty_op.operand); + + const src_ty = func.typeOf(ty_op.operand); + const dst_ty = ty_op.ty.toType(); + + const src_reg, const src_lock = try func.promoteReg(src_ty, operand); + defer if (src_lock) |lock| func.register_manager.unlockReg(lock); + + const is_unsigned = dst_ty.isUnsignedInt(zcu); + const src_bits = src_ty.bitSize(zcu); + const dst_bits = dst_ty.bitSize(zcu); + + switch (src_bits) { + 32, 64 => {}, + else => try func.truncateRegister(src_ty, src_reg), + } + + const int_zcu: Mir.FcvtOp = switch (src_bits) { + 8, 16, 32 => if (is_unsigned) .wu else .w, + 64 => if (is_unsigned) .lu else .l, + else => return func.fail("TODO: airFloatFromInt src size: {d}", .{src_bits}), + }; + + const float_zcu: enum { s, d } = switch (dst_bits) { + 32 => .s, + 64 => .d, + else => return func.fail("TODO: airFloatFromInt dst size {d}", .{dst_bits}), + }; + + const dst_reg, const dst_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(dst_lock); + + _ = try func.addInst(.{ + .tag = switch (float_zcu) { + .s => switch (int_zcu) { + .l => .fcvtsl, + .lu => .fcvtslu, + .w => .fcvtsw, + .wu => .fcvtswu, + }, + .d => switch (int_zcu) { + .l => .fcvtdl, + .lu => .fcvtdlu, + .w => .fcvtdw, + .wu => .fcvtdwu, + }, + }, + .data = .{ .rr = .{ + .rd = dst_reg, + .rs = src_reg, + } }, + }); + + break :result .{ .register = dst_reg }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIntFromFloat(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const pt = func.pt; + const zcu = pt.zcu; + + const operand = try func.resolveInst(ty_op.operand); + const src_ty = func.typeOf(ty_op.operand); + const dst_ty = ty_op.ty.toType(); + + const is_unsigned = dst_ty.isUnsignedInt(zcu); + const src_bits = src_ty.bitSize(zcu); + const dst_bits = dst_ty.bitSize(zcu); + + const float_zcu: enum { s, d } = switch (src_bits) { + 32 => .s, + 64 => .d, + else => return func.fail("TODO: airIntFromFloat src size {d}", .{src_bits}), + }; + + const int_zcu: Mir.FcvtOp = switch (dst_bits) { + 32 => if (is_unsigned) .wu else .w, + 8, 16, 64 => if (is_unsigned) .lu else .l, + else => return func.fail("TODO: airIntFromFloat dst size: {d}", .{dst_bits}), + }; + + const src_reg, const src_lock = try func.promoteReg(src_ty, operand); + defer if (src_lock) |lock| func.register_manager.unlockReg(lock); + + const dst_reg, const dst_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(dst_lock); + + _ = try func.addInst(.{ + .tag = switch (float_zcu) { + .s => switch (int_zcu) { + .l => .fcvtls, + .lu => .fcvtlus, + .w => .fcvtws, + .wu => .fcvtwus, + }, + .d => switch (int_zcu) { + .l => .fcvtld, + .lu => .fcvtlud, + .w => .fcvtwd, + .wu => .fcvtwud, + }, + }, + .data = .{ .rr = .{ + .rd = dst_reg, + .rs = src_reg, + } }, + }); + + break :result .{ .register = dst_reg }; + }; + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airCmpxchg(func: *Func, inst: Air.Inst.Index, strength: enum { weak, strong }) !void { + _ = strength; // TODO: do something with this + + const pt = func.pt; + const zcu = pt.zcu; + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + + const ptr_ty = func.typeOf(extra.ptr); + const val_ty = func.typeOf(extra.expected_value); + const val_abi_size: u32 = @intCast(val_ty.abiSize(pt.zcu)); + + switch (val_abi_size) { + 1, 2, 4, 8 => {}, + else => return func.fail("TODO: airCmpxchg Int size {}", .{val_abi_size}), + } + + const lr_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.successOrder()) { + .unordered, + => unreachable, + + .monotonic, + .release, + => .{ .aq = .none, .rl = .none }, + .acquire, + .acq_rel, + => .{ .aq = .aq, .rl = .none }, + .seq_cst => .{ .aq = .aq, .rl = .rl }, + }; + + const sc_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.failureOrder()) { + .unordered, + .release, + .acq_rel, + => unreachable, + + .monotonic, + .acquire, + .seq_cst, + => switch (extra.successOrder()) { + .release, + .seq_cst, + => .{ .aq = .none, .rl = .rl }, + else => .{ .aq = .none, .rl = .none }, + }, + }; + + const ptr_mcv = try func.resolveInst(extra.ptr); + const ptr_reg, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); + defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); + + const exp_mcv = try func.resolveInst(extra.expected_value); + const exp_reg, const exp_lock = try func.promoteReg(val_ty, exp_mcv); + defer if (exp_lock) |lock| func.register_manager.unlockReg(lock); + try func.truncateRegister(val_ty, exp_reg); + + const new_mcv = try func.resolveInst(extra.new_value); + const new_reg, const new_lock = try func.promoteReg(val_ty, new_mcv); + defer if (new_lock) |lock| func.register_manager.unlockReg(lock); + try func.truncateRegister(val_ty, new_reg); + + const branch_reg, const branch_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(branch_lock); + + const fallthrough_reg, const fallthrough_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(fallthrough_lock); + + const jump_back = try func.addInst(.{ + .tag = if (val_ty.bitSize(zcu) <= 32) .lrw else .lrd, + .data = .{ .amo = .{ + .aq = lr_order.aq, + .rl = lr_order.rl, + .rd = branch_reg, + .rs1 = ptr_reg, + .rs2 = .zero, + } }, + }); + try func.truncateRegister(val_ty, branch_reg); + + const jump_forward = try func.addInst(.{ + .tag = .bne, + .data = .{ .b_type = .{ + .rs1 = branch_reg, + .rs2 = exp_reg, + .inst = undefined, + } }, + }); + + _ = try func.addInst(.{ + .tag = if (val_ty.bitSize(zcu) <= 32) .scw else .scd, + .data = .{ .amo = .{ + .aq = sc_order.aq, + .rl = sc_order.rl, + .rd = fallthrough_reg, + .rs1 = ptr_reg, + .rs2 = new_reg, + } }, + }); + try func.truncateRegister(Type.bool, fallthrough_reg); + + _ = try func.addInst(.{ + .tag = .bne, + .data = .{ .b_type = .{ + .rs1 = fallthrough_reg, + .rs2 = .zero, + .inst = jump_back, + } }, + }); + + func.performReloc(jump_forward); + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, false); + + const tmp_reg, const tmp_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(tmp_lock); + + try func.genBinOp( + .cmp_neq, + .{ .register = branch_reg }, + val_ty, + .{ .register = exp_reg }, + val_ty, + tmp_reg, + ); + + try func.genCopy(val_ty, dst_mcv, .{ .register = branch_reg }); + try func.genCopy( + Type.bool, + dst_mcv.address().offset(@intCast(val_abi_size)).deref(), + .{ .register = tmp_reg }, + ); + + break :result dst_mcv; + }; + + return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); +} + +fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; + + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const op = extra.op(); + const order = extra.ordering(); + + const ptr_ty = func.typeOf(pl_op.operand); + const ptr_mcv = try func.resolveInst(pl_op.operand); + + const val_ty = func.typeOf(extra.operand); + const val_size = val_ty.abiSize(zcu); + const val_mcv = try func.resolveInst(extra.operand); + + if (!math.isPowerOfTwo(val_size)) + return func.fail("TODO: airAtomicRmw non-pow 2", .{}); + + switch (val_ty.zigTypeTag(pt.zcu)) { + .@"enum", .int => {}, + inline .bool, .float, .pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}), + else => unreachable, + } + + const method: enum { amo, loop } = switch (val_size) { + 1, 2 => .loop, + 4, 8 => .amo, + else => unreachable, + }; + + const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); + defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); + + const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv); + defer if (val_lock) |lock| func.register_manager.unlockReg(lock); + + const result_mcv = try func.allocRegOrMem(val_ty, inst, true); + assert(result_mcv == .register); // should fit into 8 bytes + const result_reg = result_mcv.register; + + const aq, const rl = switch (order) { + .unordered => unreachable, + .monotonic => .{ false, false }, + .acquire => .{ true, false }, + .release => .{ false, true }, + .acq_rel => .{ true, true }, + .seq_cst => .{ true, true }, + }; + + switch (method) { + .amo => { + const is_d = val_ty.abiSize(zcu) == 8; + const is_un = val_ty.isUnsignedInt(zcu); + + const mnem: Mnemonic = switch (op) { + // zig fmt: off + .Xchg => if (is_d) .amoswapd else .amoswapw, + .Add => if (is_d) .amoaddd else .amoaddw, + .And => if (is_d) .amoandd else .amoandw, + .Or => if (is_d) .amoord else .amoorw, + .Xor => if (is_d) .amoxord else .amoxorw, + .Max => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw, + .Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw, + else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}), + // zig fmt: on + }; + + _ = try func.addInst(.{ + .tag = mnem, + .data = .{ .amo = .{ + .rd = result_reg, + .rs1 = ptr_register, + .rs2 = val_register, + .aq = if (aq) .aq else .none, + .rl = if (rl) .rl else .none, + } }, + }); + }, + .loop => { + // where we'll jump back when the sc fails + const jump_back = try func.addInst(.{ + .tag = .lrw, + .data = .{ .amo = .{ + .rd = result_reg, + .rs1 = ptr_register, + .rs2 = .zero, + .aq = if (aq) .aq else .none, + .rl = if (rl) .rl else .none, + } }, + }); + + const after_reg, const after_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(after_lock); + + switch (op) { + .Add, .Sub => |tag| { + _ = try func.genBinOp( + switch (tag) { + .Add => .add, + .Sub => .sub, + else => unreachable, + }, + .{ .register = result_reg }, + val_ty, + .{ .register = val_register }, + val_ty, + after_reg, + ); + }, + + else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}), + } + + _ = try func.addInst(.{ + .tag = .scw, + .data = .{ .amo = .{ + .rd = after_reg, + .rs1 = ptr_register, + .rs2 = after_reg, + .aq = if (aq) .aq else .none, + .rl = if (rl) .rl else .none, + } }, + }); + + _ = try func.addInst(.{ + .tag = .bne, + .data = .{ .b_type = .{ + .inst = jump_back, + .rs1 = after_reg, + .rs2 = .zero, + } }, + }); + }, + } + break :result result_mcv; + }; + + return func.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); +} + +fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; + const order: std.builtin.AtomicOrder = atomic_load.order; + + const ptr_ty = func.typeOf(atomic_load.ptr); + const elem_ty = ptr_ty.childType(zcu); + const ptr_mcv = try func.resolveInst(atomic_load.ptr); + + const bit_size = elem_ty.bitSize(zcu); + if (bit_size > 64) return func.fail("TODO: airAtomicLoad > 64 bits", .{}); + + const result_mcv: MCValue = if (func.liveness.isUnused(inst)) + .{ .register = .zero } + else + try func.allocRegOrMem(elem_ty, inst, true); + assert(result_mcv == .register); // should be less than 8 bytes + + if (order == .seq_cst) { + _ = try func.addInst(.{ + .tag = .fence, + .data = .{ .fence = .{ + .pred = .rw, + .succ = .rw, + } }, + }); + } + + try func.load(result_mcv, ptr_mcv, ptr_ty); + + switch (order) { + // Don't guarantee other memory operations to be ordered after the load. + .unordered, .monotonic => {}, + // Make sure all previous reads happen before any reading or writing occurs. + .acquire, .seq_cst => { + _ = try func.addInst(.{ + .tag = .fence, + .data = .{ .fence = .{ + .pred = .r, + .succ = .rw, + } }, + }); + }, + else => unreachable, + } + + return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none }); +} + +fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ptr_ty = func.typeOf(bin_op.lhs); + const ptr_mcv = try func.resolveInst(bin_op.lhs); + + const val_ty = func.typeOf(bin_op.rhs); + const val_mcv = try func.resolveInst(bin_op.rhs); + + const bit_size = val_ty.bitSize(func.pt.zcu); + if (bit_size > 64) return func.fail("TODO: airAtomicStore > 64 bits", .{}); + + switch (order) { + .unordered, .monotonic => {}, + .release, .seq_cst => { + _ = try func.addInst(.{ + .tag = .fence, + .data = .{ .fence = .{ + .pred = .rw, + .succ = .w, + } }, + }); + }, + else => unreachable, + } + + try func.store(ptr_mcv, val_mcv, ptr_ty); + + if (order == .seq_cst) { + _ = try func.addInst(.{ + .tag = .fence, + .data = .{ .fence = .{ + .pred = .rw, + .succ = .rw, + } }, + }); + } + + return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void { + const pt = func.pt; + const zcu = pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + result: { + if (!safety and (try func.resolveInst(bin_op.rhs)) == .undef) break :result; + + const dst_ptr = try func.resolveInst(bin_op.lhs); + const dst_ptr_ty = func.typeOf(bin_op.lhs); + const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (dst_ptr_lock) |lock| func.register_manager.unlockReg(lock); + + const src_val = try func.resolveInst(bin_op.rhs); + const elem_ty = func.typeOf(bin_op.rhs); + const src_val_lock: ?RegisterLock = switch (src_val) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (src_val_lock) |lock| func.register_manager.unlockReg(lock); + + const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu)); + + if (elem_abi_size == 1) { + const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { + // TODO: this only handles slices stored in the stack + .slice => dst_ptr, + .one => dst_ptr, + .c, .many => unreachable, + }; + const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { + // TODO: this only handles slices stored in the stack + .slice => dst_ptr.address().offset(8).deref(), + .one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) }, + .c, .many => unreachable, + }; + const len_lock: ?RegisterLock = switch (len) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (len_lock) |lock| func.register_manager.unlockReg(lock); + + try func.genInlineMemset(ptr, src_val, len); + break :result; + } + + // Store the first element, and then rely on memcpy copying forwards. + // Length zero requires a runtime check - so we handle arrays specially + // here to elide it. + switch (dst_ptr_ty.ptrSize(zcu)) { + .slice => return func.fail("TODO: airMemset Slices", .{}), + .one => { + const elem_ptr_ty = try pt.singleMutPtrType(elem_ty); + + const len = dst_ptr_ty.childType(zcu).arrayLen(zcu); + + assert(len != 0); // prevented by Sema + try func.store(dst_ptr, src_val, elem_ptr_ty); + + const second_elem_ptr_reg, const second_elem_ptr_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(second_elem_ptr_lock); + + const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; + + try func.genSetReg(Type.u64, second_elem_ptr_reg, .{ .register_offset = .{ + .reg = try func.copyToTmpRegister(Type.u64, dst_ptr), + .off = elem_abi_size, + } }); + + const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; + try func.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); + }, + .c, .many => unreachable, + } + } + return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMemcpy(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const dst_ptr = try func.resolveInst(bin_op.lhs); + const src_ptr = try func.resolveInst(bin_op.rhs); + + const dst_ty = func.typeOf(bin_op.lhs); + + const len_mcv: MCValue = switch (dst_ty.ptrSize(zcu)) { + .slice => len: { + const len_reg, const len_lock = try func.allocReg(.int); + defer func.register_manager.unlockReg(len_lock); + + const elem_size = dst_ty.childType(zcu).abiSize(zcu); + try func.genBinOp( + .mul, + .{ .immediate = elem_size }, + Type.u64, + dst_ptr.address().offset(8).deref(), + Type.u64, + len_reg, + ); + break :len .{ .register = len_reg }; + }, + .one => len: { + const array_ty = dst_ty.childType(zcu); + break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) }; + }, + else => |size| return func.fail("TODO: airMemcpy size {s}", .{@tagName(size)}), + }; + const len_lock: ?RegisterLock = switch (len_mcv) { + .register => |reg| func.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (len_lock) |lock| func.register_manager.unlockReg(lock); + + try func.genInlineMemcpy(dst_ptr, src_ptr, len_mcv); + + return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMemmove(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement airMemmove for riscv64", .{}); +} + +fn airTagName(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + + const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { + const enum_ty = func.typeOf(un_op); + + // TODO: work out the bugs + if (true) return func.fail("TODO: airTagName", .{}); + + const param_regs = abi.Registers.Integer.function_arg_regs; + const dst_mcv = try func.allocRegOrMem(Type.u64, inst, false); + try func.genSetReg(Type.u64, param_regs[0], dst_mcv.address()); + + const operand = try func.resolveInst(un_op); + try func.genSetReg(enum_ty, param_regs[1], operand); + + const lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() }; + const elf_file = func.bin_file.cast(link.File.Elf).?; + const zo = elf_file.zigObjectPtr().?; + const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err| + return func.fail("{s} creating lazy symbol", .{@errorName(err)}); + + if (func.mod.pic) { + return func.fail("TODO: airTagName pic", .{}); + } else { + try func.genSetReg(Type.u64, .ra, .{ .load_symbol = .{ .sym = sym_index } }); + _ = try func.addInst(.{ + .tag = .jalr, + .data = .{ .i_type = .{ + .rd = .ra, + .rs1 = .ra, + .imm12 = Immediate.s(0), + } }, + }); + } + + break :result dst_mcv; + }; + return func.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airErrorName(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO: airErrorName", .{}); +} + +fn airSplat(func: *Func, inst: Air.Inst.Index) !void { + const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSplat for riscv64", .{}); + return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSelect(func: *Func, inst: Air.Inst.Index) !void { + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = func.air.extraData(Air.Bin, pl_op.payload).data; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSelect for riscv64", .{}); + return func.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); +} + +fn airShuffleOne(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement airShuffleOne for riscv64", .{}); +} + +fn airShuffleTwo(func: *Func, inst: Air.Inst.Index) !void { + _ = inst; + return func.fail("TODO implement airShuffleTwo for riscv64", .{}); +} + +fn airReduce(func: *Func, inst: Air.Inst.Index) !void { + const reduce = func.air.instructions.items(.data)[@intFromEnum(inst)].reduce; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airReduce for riscv64", .{}); + return func.finishAir(inst, result, .{ reduce.operand, .none, .none }); +} + +fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void { + const pt = func.pt; + const zcu = pt.zcu; + const result_ty = func.typeOfIndex(inst); + const len: usize = @intCast(result_ty.arrayLen(zcu)); + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]); + + const result: MCValue = result: { + switch (result_ty.zigTypeTag(zcu)) { + .@"struct" => { + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu)); + if (result_ty.containerLayout(zcu) == .@"packed") { + const struct_obj = zcu.typeToStruct(result_ty).?; + try func.genInlineMemset( + .{ .lea_frame = .{ .index = frame_index } }, + .{ .immediate = 0 }, + .{ .immediate = result_ty.abiSize(zcu) }, + ); + + for (elements, 0..) |elem, elem_i_usize| { + const elem_i: u32 = @intCast(elem_i_usize); + if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; + + const elem_ty = result_ty.fieldType(elem_i, zcu); + const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu)); + if (elem_bit_size > 64) { + return func.fail( + "TODO airAggregateInit implement packed structs with large fields", + .{}, + ); + } + + const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); + const elem_abi_bits = elem_abi_size * 8; + const elem_off = zcu.structPackedFieldBitOffset(struct_obj, elem_i); + const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size); + const elem_bit_off = elem_off % elem_abi_bits; + const elem_mcv = try func.resolveInst(elem); + + _ = elem_byte_off; + _ = elem_bit_off; + + const elem_lock = switch (elem_mcv) { + .register => |reg| func.register_manager.lockReg(reg), + .immediate => |imm| lock: { + if (imm == 0) continue; + break :lock null; + }, + else => null, + }; + defer if (elem_lock) |lock| func.register_manager.unlockReg(lock); + + return func.fail("TODO: airAggregateInit packed structs", .{}); + } + } else for (elements, 0..) |elem, elem_i| { + if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; + + const elem_ty = result_ty.fieldType(elem_i, zcu); + const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu)); + const elem_mcv = try func.resolveInst(elem); + try func.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv); + } + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + .array => { + const elem_ty = result_ty.childType(zcu); + const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu)); + const elem_size: u32 = @intCast(elem_ty.abiSize(zcu)); + + for (elements, 0..) |elem, elem_i| { + const elem_mcv = try func.resolveInst(elem); + const elem_off: i32 = @intCast(elem_size * elem_i); + try func.genSetMem( + .{ .frame = frame_index }, + elem_off, + elem_ty, + elem_mcv, + ); + } + if (result_ty.sentinel(zcu)) |sentinel| try func.genSetMem( + .{ .frame = frame_index }, + @intCast(elem_size * elements.len), + elem_ty, + try func.genTypedValue(sentinel), + ); + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + else => return func.fail("TODO: airAggregate {f}", .{result_ty.fmt(pt)}), + } + }; + + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); + @memcpy(buf[0..elements.len], elements); + return func.finishAir(inst, result, buf); + } + var bt = func.liveness.iterateBigTomb(inst); + for (elements) |elem| try func.feed(&bt, elem); + return func.finishAirResult(inst, result); +} + +fn airUnionInit(func: *Func, inst: Air.Inst.Index) !void { + const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = func.air.extraData(Air.UnionInit, ty_pl.payload).data; + _ = extra; + return func.fail("TODO implement airUnionInit for riscv64", .{}); + // return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); +} + +fn airPrefetch(func: *Func, inst: Air.Inst.Index) !void { + const prefetch = func.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; + // TODO: RISC-V does have prefetch instruction variants. + // see here: https://raw.githubusercontent.com/riscv/riscv-CMOs/master/specifications/cmobase-v1.0.1.pdf + return func.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none }); +} + +fn airMulAdd(func: *Func, inst: Air.Inst.Index) !void { + const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = func.air.extraData(Air.Bin, pl_op.payload).data; + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else { + return func.fail("TODO implement airMulAdd for riscv64", .{}); + }; + return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand }); +} + +fn resolveInst(func: *Func, ref: Air.Inst.Ref) InnerError!MCValue { + const pt = func.pt; + const zcu = pt.zcu; + + // If the type has no codegen bits, no need to store it. + const inst_ty = func.typeOf(ref); + if (!inst_ty.hasRuntimeBits(zcu)) + return .none; + + const mcv = if (ref.toIndex()) |inst| mcv: { + break :mcv func.inst_tracking.getPtr(inst).?.short; + } else mcv: { + const ip_index = ref.toInterned().?; + const gop = try func.const_tracking.getOrPut(func.gpa, ip_index); + if (!gop.found_existing) gop.value_ptr.* = InstTracking.init( + try func.genTypedValue(Value.fromInterned(ip_index)), + ); + break :mcv gop.value_ptr.short; + }; + + return mcv; +} + +fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking { + const tracking = func.inst_tracking.getPtr(inst).?; + return switch (tracking.short) { + .none, .unreach, .dead => unreachable, + else => tracking, + }; +} + +fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { + const pt = func.pt; + + const lf = func.bin_file; + const src_loc = func.src_loc; + + const result: codegen.GenResult = if (val.isUndef(pt.zcu)) + switch (try lf.lowerUav(pt, val.toIntern(), .none, src_loc)) { + .sym_index => |sym_index| .{ .mcv = .{ .load_symbol = sym_index } }, + .fail => |em| .{ .fail = em }, + } + else + try codegen.genTypedValue(lf, pt, src_loc, val, func.target); + const mcv: MCValue = switch (result) { + .mcv => |mcv| switch (mcv) { + .none => .none, + .undef => unreachable, + .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } }, + .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, + .immediate => |imm| .{ .immediate = imm }, + .memory => |addr| .{ .memory = addr }, + .load_got, .load_direct, .lea_direct => { + return func.fail("TODO: genTypedValue {s}", .{@tagName(mcv)}); + }, + }, + .fail => |msg| return func.failMsg(msg), + }; + return mcv; +} + +const CallMCValues = struct { + args: []MCValue, + return_value: InstTracking, + stack_byte_count: u31, + stack_align: Alignment, + + fn deinit(call: *CallMCValues, func: *Func) void { + func.gpa.free(call.args); + call.* = undefined; + } +}; + +/// Caller must call `CallMCValues.deinit`. +fn resolveCallingConventionValues( + func: *Func, + fn_info: InternPool.Key.FuncType, + var_args: []const Type, +) !CallMCValues { + const pt = func.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const param_types = try func.gpa.alloc(Type, fn_info.param_types.len + var_args.len); + defer func.gpa.free(param_types); + + for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| { + dest.* = Type.fromInterned(src); + } + for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty| + param_ty.* = func.promoteVarArg(arg_ty); + + const cc = fn_info.cc; + var result: CallMCValues = .{ + .args = try func.gpa.alloc(MCValue, param_types.len), + // These undefined values must be populated before returning from this function. + .return_value = undefined, + .stack_byte_count = 0, + .stack_align = undefined, + }; + errdefer func.gpa.free(result.args); + + const ret_ty = Type.fromInterned(fn_info.return_type); + + switch (cc) { + .naked => { + assert(result.args.len == 0); + result.return_value = InstTracking.init(.unreach); + result.stack_align = .@"8"; + }, + .riscv64_lp64, .auto => { + if (result.args.len > 8) { + return func.fail("RISC-V calling convention does not support more than 8 arguments", .{}); + } + + var ret_int_reg_i: u32 = 0; + var param_int_reg_i: u32 = 0; + + result.stack_align = .@"16"; + + // Return values + if (ret_ty.zigTypeTag(zcu) == .noreturn) { + result.return_value = InstTracking.init(.unreach); + } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + result.return_value = InstTracking.init(.none); + } else { + var ret_tracking: [2]InstTracking = undefined; + var ret_tracking_i: usize = 0; + var ret_float_reg_i: usize = 0; + + const classes = mem.sliceTo(&abi.classifySystem(ret_ty, zcu), .none); + + for (classes) |class| switch (class) { + .integer => { + const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i]; + ret_int_reg_i += 1; + + ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg }); + ret_tracking_i += 1; + }, + .float => { + const ret_float_reg = abi.Registers.Float.function_ret_regs[ret_float_reg_i]; + ret_float_reg_i += 1; + + ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_float_reg }); + ret_tracking_i += 1; + }, + .memory => { + const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i]; + ret_int_reg_i += 1; + const ret_indirect_reg = abi.Registers.Integer.function_arg_regs[param_int_reg_i]; + param_int_reg_i += 1; + + ret_tracking[ret_tracking_i] = .{ + .short = .{ .indirect = .{ .reg = ret_int_reg } }, + .long = .{ .indirect = .{ .reg = ret_indirect_reg } }, + }; + ret_tracking_i += 1; + }, + else => return func.fail("TODO: C calling convention return class {}", .{class}), + }; + + result.return_value = switch (ret_tracking_i) { + else => return func.fail("ty {f} took {} tracking return indices", .{ ret_ty.fmt(pt), ret_tracking_i }), + 1 => ret_tracking[0], + 2 => InstTracking.init(.{ .register_pair = .{ + ret_tracking[0].short.register, ret_tracking[1].short.register, + } }), + }; + } + + var param_float_reg_i: usize = 0; + + for (param_types, result.args) |ty, *arg| { + if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) { + assert(cc == .auto); + arg.* = .none; + continue; + } + + var arg_mcv: [2]MCValue = undefined; + var arg_mcv_i: usize = 0; + + const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none); + + for (classes) |class| switch (class) { + .integer => { + const param_int_regs = abi.Registers.Integer.function_arg_regs; + if (param_int_reg_i >= param_int_regs.len) break; + + const param_int_reg = param_int_regs[param_int_reg_i]; + param_int_reg_i += 1; + + arg_mcv[arg_mcv_i] = .{ .register = param_int_reg }; + arg_mcv_i += 1; + }, + .float => { + const param_float_regs = abi.Registers.Float.function_arg_regs; + if (param_float_reg_i >= param_float_regs.len) break; + + const param_float_reg = param_float_regs[param_float_reg_i]; + param_float_reg_i += 1; + + arg_mcv[arg_mcv_i] = .{ .register = param_float_reg }; + arg_mcv_i += 1; + }, + .memory => { + const param_int_regs = abi.Registers.Integer.function_arg_regs; + + const param_int_reg = param_int_regs[param_int_reg_i]; + param_int_reg_i += 1; + + arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } }; + arg_mcv_i += 1; + }, + else => return func.fail("TODO: C calling convention arg class {}", .{class}), + } else { + arg.* = switch (arg_mcv_i) { + else => return func.fail("ty {f} took {} tracking arg indices", .{ ty.fmt(pt), arg_mcv_i }), + 1 => arg_mcv[0], + 2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } }, + }; + continue; + } + + return func.fail("TODO: pass args by stack", .{}); + } + }, + else => return func.fail("TODO implement function parameters for {} on riscv64", .{cc}), + } + + result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count)); + return result; +} + +fn wantSafety(func: *Func) bool { + return switch (func.mod.optimize_mode) { + .Debug => true, + .ReleaseSafe => true, + .ReleaseFast => false, + .ReleaseSmall => false, + }; +} + +fn fail(func: *const Func, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = func.pt.zcu; + switch (func.owner) { + .nav_index => |i| return zcu.codegenFail(i, format, args), + .lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args), + } + return error.CodegenFail; +} + +fn failMsg(func: *const Func, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = func.pt.zcu; + switch (func.owner) { + .nav_index => |i| return zcu.codegenFailMsg(i, msg), + .lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg), + } + return error.CodegenFail; +} + +fn parseRegName(name: []const u8) ?Register { + // The `fp` alias for `s0` is awkward to fit into the current `Register` scheme, so for now we + // special-case it here. + if (std.mem.eql(u8, name, "fp")) return .s0; + + return std.meta.stringToEnum(Register, name); +} + +fn typeOf(func: *Func, inst: Air.Inst.Ref) Type { + return func.air.typeOf(inst, &func.pt.zcu.intern_pool); +} + +fn typeOfIndex(func: *Func, inst: Air.Inst.Index) Type { + const zcu = func.pt.zcu; + return switch (func.air.instructions.items(.tag)[@intFromEnum(inst)]) { + .loop_switch_br => func.typeOf(func.air.unwrapSwitch(inst).operand), + else => func.air.typeOfIndex(inst, &zcu.intern_pool), + }; +} + +fn hasFeature(func: *Func, feature: Target.riscv.Feature) bool { + return func.target.cpu.has(.riscv, feature); +} + +pub fn errUnionPayloadOffset(payload_ty: Type, zcu: *Zcu) u64 { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0; + const payload_align = payload_ty.abiAlignment(zcu); + const error_align = Type.anyerror.abiAlignment(zcu); + if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return 0; + } else { + return payload_align.forward(Type.anyerror.abiSize(zcu)); + } +} + +pub fn errUnionErrorOffset(payload_ty: Type, zcu: *Zcu) u64 { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0; + const payload_align = payload_ty.abiAlignment(zcu); + const error_align = Type.anyerror.abiAlignment(zcu); + if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return error_align.forward(payload_ty.abiSize(zcu)); + } else { + return 0; + } +} + +fn promoteInt(func: *Func, ty: Type) Type { + const pt = func.pt; + const zcu = pt.zcu; + const int_info: InternPool.Key.IntType = switch (ty.toIntern()) { + .bool_type => .{ .signedness = .unsigned, .bits = 1 }, + else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty, + }; + for ([_]Type{ + Type.c_int, Type.c_uint, + Type.c_long, Type.c_ulong, + Type.c_longlong, Type.c_ulonglong, + }) |promote_ty| { + const promote_info = promote_ty.intInfo(zcu); + if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue; + if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and + promote_info.signedness == .signed) <= promote_info.bits) return promote_ty; + } + return ty; +} + +fn promoteVarArg(func: *Func, ty: Type) Type { + if (!ty.isRuntimeFloat()) return func.promoteInt(ty); + switch (ty.floatBits(func.target)) { + 32, 64 => return Type.f64, + else => |float_bits| { + assert(float_bits == func.target.cTypeBitSize(.longdouble)); + return Type.c_longdouble; + }, + } +} diff --git a/src/codegen/riscv64/Emit.zig b/src/codegen/riscv64/Emit.zig new file mode 100644 index 0000000000000000000000000000000000000000..64a476007cf9881e5101c5c945158252b1ba5b86 --- /dev/null +++ b/src/codegen/riscv64/Emit.zig @@ -0,0 +1,222 @@ +//! This file contains the functionality for emitting RISC-V MIR as machine code + +bin_file: *link.File, +lower: Lower, +debug_output: link.File.DebugInfoOutput, +w: *std.Io.Writer, + +prev_di_line: u32, +prev_di_column: u32, +/// Relative to the beginning of `code`. +prev_di_pc: usize, + +code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty, +relocs: std.ArrayListUnmanaged(Reloc) = .empty, + +pub const Error = Lower.Error || std.Io.Writer.Error || error{ + EmitFail, +}; + +pub fn emitMir(emit: *Emit) Error!void { + const gpa = emit.bin_file.comp.gpa; + log.debug("mir instruction len: {}", .{emit.lower.mir.instructions.len}); + for (0..emit.lower.mir.instructions.len) |mir_i| { + const mir_index: Mir.Inst.Index = @intCast(mir_i); + try emit.code_offset_mapping.putNoClobber( + emit.lower.allocator, + mir_index, + @intCast(emit.w.end), + ); + const lowered = try emit.lower.lowerMir(mir_index, .{ .allow_frame_locs = true }); + var lowered_relocs = lowered.relocs; + for (lowered.insts, 0..) |lowered_inst, lowered_index| { + const start_offset: u32 = @intCast(emit.w.end); + try emit.w.writeInt(u32, lowered_inst.toU32(), .little); + + while (lowered_relocs.len > 0 and + lowered_relocs[0].lowered_inst_index == lowered_index) : ({ + lowered_relocs = lowered_relocs[1..]; + }) switch (lowered_relocs[0].target) { + .inst => |target| try emit.relocs.append(emit.lower.allocator, .{ + .source = start_offset, + .target = target, + .offset = 0, + .fmt = std.meta.activeTag(lowered_inst), + }), + .load_symbol_reloc => |symbol| { + const elf_file = emit.bin_file.cast(.elf).?; + const zo = elf_file.zigObjectPtr().?; + + const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; + const sym = zo.symbol(symbol.sym_index); + + if (emit.lower.pic) { + return emit.fail("know when to emit GOT relocation for symbol '{s}'", .{sym.name(elf_file)}); + } + + const hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); + const lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); + + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset, + .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | hi_r_type, + .r_addend = 0, + }, zo); + + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset + 4, + .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | lo_r_type, + .r_addend = 0, + }, zo); + }, + .load_tlv_reloc => |symbol| { + const elf_file = emit.bin_file.cast(.elf).?; + const zo = elf_file.zigObjectPtr().?; + + const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; + + const R_RISCV = std.elf.R_RISCV; + + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset, + .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | @intFromEnum(R_RISCV.TPREL_HI20), + .r_addend = 0, + }, zo); + + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset + 4, + .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | @intFromEnum(R_RISCV.TPREL_ADD), + .r_addend = 0, + }, zo); + + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset + 8, + .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | @intFromEnum(R_RISCV.TPREL_LO12_I), + .r_addend = 0, + }, zo); + }, + .call_extern_fn_reloc => |symbol| { + const elf_file = emit.bin_file.cast(.elf).?; + const zo = elf_file.zigObjectPtr().?; + const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; + + const r_type: u32 = @intFromEnum(std.elf.R_RISCV.CALL_PLT); + + try atom_ptr.addReloc(gpa, .{ + .r_offset = start_offset, + .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | r_type, + .r_addend = 0, + }, zo); + }, + }; + } + std.debug.assert(lowered_relocs.len == 0); + + if (lowered.insts.len == 0) { + const mir_inst = emit.lower.mir.instructions.get(mir_index); + switch (mir_inst.tag) { + else => unreachable, + .pseudo_dbg_prologue_end => { + switch (emit.debug_output) { + .dwarf => |dw| { + try dw.setPrologueEnd(); + log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{ + emit.prev_di_line, emit.prev_di_column, + }); + try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); + }, + .none => {}, + } + }, + .pseudo_dbg_line_column => try emit.dbgAdvancePCAndLine( + mir_inst.data.pseudo_dbg_line_column.line, + mir_inst.data.pseudo_dbg_line_column.column, + ), + .pseudo_dbg_epilogue_begin => { + switch (emit.debug_output) { + .dwarf => |dw| { + try dw.setEpilogueBegin(); + log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{ + emit.prev_di_line, emit.prev_di_column, + }); + try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); + }, + .none => {}, + } + }, + .pseudo_dead => {}, + } + } + } + try emit.fixupRelocs(); +} + +pub fn deinit(emit: *Emit) void { + emit.relocs.deinit(emit.lower.allocator); + emit.code_offset_mapping.deinit(emit.lower.allocator); + emit.* = undefined; +} + +const Reloc = struct { + /// Offset of the instruction. + source: usize, + /// Target of the relocation. + target: Mir.Inst.Index, + /// Offset of the relocation within the instruction. + offset: u32, + /// Format of the instruction, used to determine how to modify it. + fmt: encoding.Lir.Format, +}; + +fn fixupRelocs(emit: *Emit) Error!void { + for (emit.relocs.items) |reloc| { + log.debug("target inst: {f}", .{emit.lower.mir.instructions.get(reloc.target)}); + const target = emit.code_offset_mapping.get(reloc.target) orelse + return emit.fail("relocation target not found!", .{}); + + const disp = @as(i32, @intCast(target)) - @as(i32, @intCast(reloc.source)); + const code = emit.w.buffered()[reloc.source + reloc.offset ..][0..4]; + + switch (reloc.fmt) { + .J => riscv_util.writeInstJ(code, @bitCast(disp)), + .B => riscv_util.writeInstB(code, @bitCast(disp)), + else => return emit.fail("tried to reloc format type {s}", .{@tagName(reloc.fmt)}), + } + } +} + +fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) Error!void { + const delta_line = @as(i33, line) - @as(i33, emit.prev_di_line); + const delta_pc: usize = emit.w.end - emit.prev_di_pc; + log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line }); + switch (emit.debug_output) { + .dwarf => |dw| { + if (column != emit.prev_di_column) try dw.setColumn(column); + if (delta_line == 0) return; // TODO: fix these edge cases. + try dw.advancePCAndLine(delta_line, delta_pc); + emit.prev_di_line = line; + emit.prev_di_column = column; + emit.prev_di_pc = emit.w.end; + }, + .none => {}, + } +} + +fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error { + return switch (emit.lower.fail(format, args)) { + error.LowerFail => error.EmitFail, + else => |e| e, + }; +} + +const link = @import("../../link.zig"); +const log = std.log.scoped(.emit); +const mem = std.mem; +const std = @import("std"); + +const Emit = @This(); +const Lower = @import("Lower.zig"); +const Mir = @import("Mir.zig"); +const riscv_util = @import("../../link/riscv.zig"); +const Elf = @import("../../link/Elf.zig"); +const encoding = @import("encoding.zig"); diff --git a/src/codegen/riscv64/Lower.zig b/src/codegen/riscv64/Lower.zig new file mode 100644 index 0000000000000000000000000000000000000000..08bb3e2b937b448492030a76d5467dec24da9e21 --- /dev/null +++ b/src/codegen/riscv64/Lower.zig @@ -0,0 +1,615 @@ +//! This file contains the functionality for lowering RISC-V MIR to Instructions + +pt: Zcu.PerThread, +output_mode: std.builtin.OutputMode, +link_mode: std.builtin.LinkMode, +pic: bool, +allocator: Allocator, +mir: Mir, +cc: std.builtin.CallingConvention, +err_msg: ?*ErrorMsg = null, +src_loc: Zcu.LazySrcLoc, +result_insts_len: u8 = undefined, +result_relocs_len: u8 = undefined, +result_insts: [ + @max( + 1, // non-pseudo instruction + abi.Registers.all_preserved.len, // spill / restore regs, + ) +]Instruction = undefined, +result_relocs: [1]Reloc = undefined, + +pub const Error = error{ + OutOfMemory, + LowerFail, + InvalidInstruction, +}; + +pub const Reloc = struct { + lowered_inst_index: u8, + target: Target, + + const Target = union(enum) { + inst: Mir.Inst.Index, + + /// Relocs the lowered_inst_index and the next instruction. + load_symbol_reloc: bits.Symbol, + /// Relocs the lowered_inst_index and the next two instructions. + load_tlv_reloc: bits.Symbol, + /// Relocs the lowered_inst_index and the next instruction. + call_extern_fn_reloc: bits.Symbol, + }; +}; + +/// The returned slice is overwritten by the next call to lowerMir. +pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct { + allow_frame_locs: bool, +}) Error!struct { + insts: []const Instruction, + relocs: []const Reloc, +} { + const pt = lower.pt; + const zcu = pt.zcu; + + lower.result_insts = undefined; + lower.result_relocs = undefined; + errdefer lower.result_insts = undefined; + errdefer lower.result_relocs = undefined; + lower.result_insts_len = 0; + lower.result_relocs_len = 0; + defer lower.result_insts_len = undefined; + defer lower.result_relocs_len = undefined; + + const inst = lower.mir.instructions.get(index); + log.debug("lowerMir {f}", .{inst}); + switch (inst.tag) { + else => try lower.generic(inst), + .pseudo_dbg_line_column, + .pseudo_dbg_epilogue_begin, + .pseudo_dbg_prologue_end, + .pseudo_dead, + => {}, + + .pseudo_load_rm, .pseudo_store_rm => { + const rm = inst.data.rm; + + const frame_loc: Mir.FrameLoc = if (options.allow_frame_locs) + rm.m.toFrameLoc(lower.mir) + else + .{ .base = .s0, .disp = 0 }; + + switch (inst.tag) { + .pseudo_load_rm => { + const dest_reg = rm.r; + const dest_reg_class = dest_reg.class(); + + const src_size = rm.m.mod.size; + const unsigned = rm.m.mod.unsigned; + + const mnem: Mnemonic = switch (dest_reg_class) { + .int => switch (src_size) { + .byte => if (unsigned) .lbu else .lb, + .hword => if (unsigned) .lhu else .lh, + .word => if (unsigned) .lwu else .lw, + .dword => .ld, + }, + .float => switch (src_size) { + .byte => unreachable, // Zig does not support 8-bit floats + .hword => return lower.fail("TODO: lowerMir pseudo_load_rm support 16-bit floats", .{}), + .word => .flw, + .dword => .fld, + }, + .vector => switch (src_size) { + .byte => .vle8v, + .hword => .vle32v, + .word => .vle32v, + .dword => .vle64v, + }, + }; + + switch (dest_reg_class) { + .int, .float => { + try lower.emit(mnem, &.{ + .{ .reg = rm.r }, + .{ .reg = frame_loc.base }, + .{ .imm = Immediate.s(frame_loc.disp) }, + }); + }, + .vector => { + assert(frame_loc.disp == 0); + try lower.emit(mnem, &.{ + .{ .reg = rm.r }, + .{ .reg = frame_loc.base }, + .{ .reg = .zero }, + }); + }, + } + }, + .pseudo_store_rm => { + const src_reg = rm.r; + const src_reg_class = src_reg.class(); + + const dest_size = rm.m.mod.size; + + const mnem: Mnemonic = switch (src_reg_class) { + .int => switch (dest_size) { + .byte => .sb, + .hword => .sh, + .word => .sw, + .dword => .sd, + }, + .float => switch (dest_size) { + .byte => unreachable, // Zig does not support 8-bit floats + .hword => return lower.fail("TODO: lowerMir pseudo_store_rm support 16-bit floats", .{}), + .word => .fsw, + .dword => .fsd, + }, + .vector => switch (dest_size) { + .byte => .vse8v, + .hword => .vse16v, + .word => .vse32v, + .dword => .vse64v, + }, + }; + + switch (src_reg_class) { + .int, .float => { + try lower.emit(mnem, &.{ + .{ .reg = frame_loc.base }, + .{ .reg = rm.r }, + .{ .imm = Immediate.s(frame_loc.disp) }, + }); + }, + .vector => { + assert(frame_loc.disp == 0); + try lower.emit(mnem, &.{ + .{ .reg = rm.r }, + .{ .reg = frame_loc.base }, + .{ .reg = .zero }, + }); + }, + } + }, + else => unreachable, + } + }, + + .pseudo_mv => { + const rr = inst.data.rr; + + const dst_class = rr.rd.class(); + const src_class = rr.rs.class(); + + switch (src_class) { + .float => switch (dst_class) { + .float => { + try lower.emit(if (lower.hasFeature(.d)) .fsgnjnd else .fsgnjns, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .reg = rr.rs }, + }); + }, + .int, .vector => return lower.fail("TODO: lowerMir pseudo_mv float -> {s}", .{@tagName(dst_class)}), + }, + .int => switch (dst_class) { + .int => { + try lower.emit(.addi, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .imm = Immediate.s(0) }, + }); + }, + .vector => { + try lower.emit(.vmvvx, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .reg = .x0 }, + }); + }, + .float => return lower.fail("TODO: lowerMir pseudo_mv int -> {s}", .{@tagName(dst_class)}), + }, + .vector => switch (dst_class) { + .int => { + try lower.emit(.vadcvv, &.{ + .{ .reg = rr.rd }, + .{ .reg = .zero }, + .{ .reg = rr.rs }, + }); + }, + .float, .vector => return lower.fail("TODO: lowerMir pseudo_mv vector -> {s}", .{@tagName(dst_class)}), + }, + } + }, + + .pseudo_j => { + const j_type = inst.data.j_type; + try lower.emit(.jal, &.{ + .{ .reg = j_type.rd }, + .{ .imm = lower.reloc(.{ .inst = j_type.inst }) }, + }); + }, + + .pseudo_spill_regs => try lower.pushPopRegList(true, inst.data.reg_list), + .pseudo_restore_regs => try lower.pushPopRegList(false, inst.data.reg_list), + + .pseudo_load_symbol => { + const payload = inst.data.reloc; + const dst_reg = payload.register; + assert(dst_reg.class() == .int); + + try lower.emit(.lui, &.{ + .{ .reg = dst_reg }, + .{ .imm = lower.reloc(.{ + .load_symbol_reloc = .{ + .atom_index = payload.atom_index, + .sym_index = payload.sym_index, + }, + }) }, + }); + + // the reloc above implies this one + try lower.emit(.addi, &.{ + .{ .reg = dst_reg }, + .{ .reg = dst_reg }, + .{ .imm = Immediate.s(0) }, + }); + }, + + .pseudo_load_tlv => { + const payload = inst.data.reloc; + const dst_reg = payload.register; + assert(dst_reg.class() == .int); + + try lower.emit(.lui, &.{ + .{ .reg = dst_reg }, + .{ .imm = lower.reloc(.{ + .load_tlv_reloc = .{ + .atom_index = payload.atom_index, + .sym_index = payload.sym_index, + }, + }) }, + }); + + try lower.emit(.add, &.{ + .{ .reg = dst_reg }, + .{ .reg = dst_reg }, + .{ .reg = .tp }, + }); + + try lower.emit(.addi, &.{ + .{ .reg = dst_reg }, + .{ .reg = dst_reg }, + .{ .imm = Immediate.s(0) }, + }); + }, + + .pseudo_lea_rm => { + const rm = inst.data.rm; + assert(rm.r.class() == .int); + + const frame: Mir.FrameLoc = if (options.allow_frame_locs) + rm.m.toFrameLoc(lower.mir) + else + .{ .base = .s0, .disp = 0 }; + + try lower.emit(.addi, &.{ + .{ .reg = rm.r }, + .{ .reg = frame.base }, + .{ .imm = Immediate.s(frame.disp) }, + }); + }, + + .pseudo_compare => { + const compare = inst.data.compare; + const op = compare.op; + + const rd = compare.rd; + const rs1 = compare.rs1; + const rs2 = compare.rs2; + + const class = rs1.class(); + const ty = compare.ty; + const size = std.math.ceilPowerOfTwo(u64, ty.bitSize(zcu)) catch { + return lower.fail("pseudo_compare size {}", .{ty.bitSize(zcu)}); + }; + + const is_unsigned = ty.isUnsignedInt(zcu); + const less_than: Mnemonic = if (is_unsigned) .sltu else .slt; + + switch (class) { + .int => switch (op) { + .eq => { + try lower.emit(.xor, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + + try lower.emit(.sltiu, &.{ + .{ .reg = rd }, + .{ .reg = rd }, + .{ .imm = Immediate.s(1) }, + }); + }, + .neq => { + try lower.emit(.xor, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + + try lower.emit(.sltu, &.{ + .{ .reg = rd }, + .{ .reg = .zero }, + .{ .reg = rd }, + }); + }, + .gt => { + try lower.emit(less_than, &.{ + .{ .reg = rd }, + .{ .reg = rs2 }, + .{ .reg = rs1 }, + }); + }, + .gte => { + try lower.emit(less_than, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + try lower.emit(.xori, &.{ + .{ .reg = rd }, + .{ .reg = rd }, + .{ .imm = Immediate.s(1) }, + }); + }, + .lt => { + try lower.emit(less_than, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + }, + .lte => { + try lower.emit(less_than, &.{ + .{ .reg = rd }, + .{ .reg = rs2 }, + .{ .reg = rs1 }, + }); + + try lower.emit(.xori, &.{ + .{ .reg = rd }, + .{ .reg = rd }, + .{ .imm = Immediate.s(1) }, + }); + }, + }, + .float => switch (op) { + // eq + .eq => { + try lower.emit(if (size == 64) .feqd else .feqs, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + }, + // !(eq) + .neq => { + try lower.emit(if (size == 64) .feqd else .feqs, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + try lower.emit(.xori, &.{ + .{ .reg = rd }, + .{ .reg = rd }, + .{ .imm = Immediate.s(1) }, + }); + }, + .lt => { + try lower.emit(if (size == 64) .fltd else .flts, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + }, + .lte => { + try lower.emit(if (size == 64) .fled else .fles, &.{ + .{ .reg = rd }, + .{ .reg = rs1 }, + .{ .reg = rs2 }, + }); + }, + .gt => { + try lower.emit(if (size == 64) .fltd else .flts, &.{ + .{ .reg = rd }, + .{ .reg = rs2 }, + .{ .reg = rs1 }, + }); + }, + .gte => { + try lower.emit(if (size == 64) .fled else .fles, &.{ + .{ .reg = rd }, + .{ .reg = rs2 }, + .{ .reg = rs1 }, + }); + }, + }, + .vector => return lower.fail("TODO: lowerMir pseudo_cmp vector", .{}), + } + }, + + .pseudo_not => { + const rr = inst.data.rr; + assert(rr.rs.class() == .int and rr.rd.class() == .int); + + // mask out any other bits that aren't the boolean + try lower.emit(.andi, &.{ + .{ .reg = rr.rs }, + .{ .reg = rr.rs }, + .{ .imm = Immediate.s(1) }, + }); + + try lower.emit(.sltiu, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .imm = Immediate.s(1) }, + }); + }, + + .pseudo_extern_fn_reloc => { + const inst_reloc = inst.data.reloc; + const link_reg = inst_reloc.register; + + try lower.emit(.auipc, &.{ + .{ .reg = link_reg }, + .{ .imm = lower.reloc( + .{ .call_extern_fn_reloc = .{ + .atom_index = inst_reloc.atom_index, + .sym_index = inst_reloc.sym_index, + } }, + ) }, + }); + + try lower.emit(.jalr, &.{ + .{ .reg = link_reg }, + .{ .reg = link_reg }, + .{ .imm = Immediate.s(0) }, + }); + }, + } + + return .{ + .insts = lower.result_insts[0..lower.result_insts_len], + .relocs = lower.result_relocs[0..lower.result_relocs_len], + }; +} + +fn generic(lower: *Lower, inst: Mir.Inst) Error!void { + const mnemonic = inst.tag; + try lower.emit(mnemonic, switch (inst.data) { + .none => &.{}, + .u_type => |u| &.{ + .{ .reg = u.rd }, + .{ .imm = u.imm20 }, + }, + .i_type => |i| &.{ + .{ .reg = i.rd }, + .{ .reg = i.rs1 }, + .{ .imm = i.imm12 }, + }, + .rr => |rr| &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + }, + .b_type => |b| &.{ + .{ .reg = b.rs1 }, + .{ .reg = b.rs2 }, + .{ .imm = lower.reloc(.{ .inst = b.inst }) }, + }, + .r_type => |r| &.{ + .{ .reg = r.rd }, + .{ .reg = r.rs1 }, + .{ .reg = r.rs2 }, + }, + .csr => |csr| &.{ + .{ .csr = csr.csr }, + .{ .reg = csr.rs1 }, + .{ .reg = csr.rd }, + }, + .amo => |amo| &.{ + .{ .reg = amo.rd }, + .{ .reg = amo.rs1 }, + .{ .reg = amo.rs2 }, + .{ .barrier = amo.rl }, + .{ .barrier = amo.aq }, + }, + .fence => |fence| &.{ + .{ .barrier = fence.succ }, + .{ .barrier = fence.pred }, + }, + else => return lower.fail("TODO: generic lower {s}", .{@tagName(inst.data)}), + }); +} + +fn emit(lower: *Lower, mnemonic: Mnemonic, ops: []const Instruction.Operand) !void { + const lir = encoding.Lir.fromMnem(mnemonic); + const inst = Instruction.fromLir(lir, ops); + + lower.result_insts[lower.result_insts_len] = inst; + lower.result_insts_len += 1; +} + +fn reloc(lower: *Lower, target: Reloc.Target) Immediate { + lower.result_relocs[lower.result_relocs_len] = .{ + .lowered_inst_index = lower.result_insts_len, + .target = target, + }; + lower.result_relocs_len += 1; + return Immediate.s(0); +} + +fn pushPopRegList(lower: *Lower, comptime spilling: bool, reg_list: Mir.RegisterList) !void { + var it = reg_list.iterator(.{ .direction = .forward }); + + var reg_i: u31 = 0; + while (it.next()) |i| { + const frame = lower.mir.frame_locs.get(@intFromEnum(bits.FrameIndex.spill_frame)); + const reg = abi.Registers.all_preserved[i]; + + const reg_class = reg.class(); + const load_inst: Mnemonic, const store_inst: Mnemonic = switch (reg_class) { + .int => .{ .ld, .sd }, + .float => .{ .fld, .fsd }, + .vector => unreachable, + }; + + if (spilling) { + try lower.emit(store_inst, &.{ + .{ .reg = frame.base }, + .{ .reg = abi.Registers.all_preserved[i] }, + .{ .imm = Immediate.s(frame.disp + reg_i) }, + }); + } else { + try lower.emit(load_inst, &.{ + .{ .reg = abi.Registers.all_preserved[i] }, + .{ .reg = frame.base }, + .{ .imm = Immediate.s(frame.disp + reg_i) }, + }); + } + + reg_i += 8; + } +} + +pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { + @branchHint(.cold); + assert(lower.err_msg == null); + lower.err_msg = try ErrorMsg.create(lower.allocator, lower.src_loc, format, args); + return error.LowerFail; +} + +fn hasFeature(lower: *Lower, feature: std.Target.riscv.Feature) bool { + return lower.pt.zcu.getTarget().cpu.has(.riscv, feature); +} + +const Lower = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.lower); + +const Allocator = std.mem.Allocator; +const ErrorMsg = Zcu.ErrorMsg; + +const link = @import("../../link.zig"); +const Air = @import("../../Air.zig"); +const Zcu = @import("../../Zcu.zig"); + +const Mir = @import("Mir.zig"); +const abi = @import("abi.zig"); +const bits = @import("bits.zig"); +const encoding = @import("encoding.zig"); + +const Mnemonic = @import("mnem.zig").Mnemonic; +const Immediate = bits.Immediate; +const Instruction = encoding.Instruction; diff --git a/src/codegen/riscv64/Mir.zig b/src/codegen/riscv64/Mir.zig new file mode 100644 index 0000000000000000000000000000000000000000..204ab6a3c06226635b1619b60c205f02f8447884 --- /dev/null +++ b/src/codegen/riscv64/Mir.zig @@ -0,0 +1,248 @@ +//! Machine Intermediate Representation. +//! This data is produced by CodeGen.zig + +instructions: std.MultiArrayList(Inst).Slice, +frame_locs: std.MultiArrayList(FrameLoc).Slice, + +pub const Inst = struct { + tag: Mnemonic, + data: Data, + + pub const Index = u32; + + pub const Data = union(enum) { + none: void, + r_type: struct { + rd: Register, + rs1: Register, + rs2: Register, + }, + i_type: struct { + rd: Register, + rs1: Register, + imm12: Immediate, + }, + s_type: struct { + rs1: Register, + rs2: Register, + imm5: Immediate, + imm7: Immediate, + }, + b_type: struct { + rs1: Register, + rs2: Register, + inst: Inst.Index, + }, + u_type: struct { + rd: Register, + imm20: Immediate, + }, + j_type: struct { + rd: Register, + inst: Inst.Index, + }, + rm: struct { + r: Register, + m: Memory, + }, + reg_list: Mir.RegisterList, + reg: Register, + rr: struct { + rd: Register, + rs: Register, + }, + compare: struct { + rd: Register, + rs1: Register, + rs2: Register, + op: enum { + eq, + neq, + gt, + gte, + lt, + lte, + }, + ty: Type, + }, + reloc: struct { + register: Register, + atom_index: u32, + sym_index: u32, + }, + fence: struct { + pred: Barrier, + succ: Barrier, + }, + amo: struct { + rd: Register, + rs1: Register, + rs2: Register, + aq: Barrier, + rl: Barrier, + }, + csr: struct { + csr: CSR, + rs1: Register, + rd: Register, + }, + pseudo_dbg_line_column: struct { + line: u32, + column: u32, + }, + }; + + pub fn format(inst: Inst, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Tag: {s}, Data: {s}", .{ @tagName(inst.tag), @tagName(inst.data) }); + } +}; + +pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { + mir.instructions.deinit(gpa); + mir.frame_locs.deinit(gpa); + mir.* = undefined; +} + +pub fn emit( + mir: Mir, + lf: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + atom_index: u32, + w: *std.Io.Writer, + debug_output: link.File.DebugInfoOutput, +) (codegen.CodeGenError || std.Io.Writer.Error)!void { + _ = atom_index; + const zcu = pt.zcu; + const comp = zcu.comp; + const gpa = comp.gpa; + const func = zcu.funcInfo(func_index); + const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?; + const nav = func.owner_nav; + const mod = zcu.navFileScope(nav).mod.?; + var e: Emit = .{ + .lower = .{ + .pt = pt, + .allocator = gpa, + .mir = mir, + .cc = fn_info.cc, + .src_loc = src_loc, + .output_mode = comp.config.output_mode, + .link_mode = comp.config.link_mode, + .pic = mod.pic, + }, + .bin_file = lf, + .debug_output = debug_output, + .w = w, + .prev_di_pc = 0, + .prev_di_line = func.lbrace_line, + .prev_di_column = func.lbrace_column, + }; + defer e.deinit(); + e.emitMir() catch |err| switch (err) { + error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, e.lower.err_msg.?), + error.InvalidInstruction => return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}), + else => |err1| return err1, + }; +} + +pub const FrameLoc = struct { + base: Register, + disp: i32, +}; + +pub const Barrier = enum(u4) { + // Fence + w = 0b0001, + r = 0b0010, + rw = 0b0011, + + // Amo + none, + aq, + rl, +}; + +pub const AmoOp = enum(u5) { + SWAP, + ADD, + AND, + OR, + XOR, + MAX, + MIN, +}; + +pub const FcvtOp = enum(u5) { + w = 0b00000, + wu = 0b00001, + l = 0b00010, + lu = 0b00011, +}; + +pub const LoadSymbolPayload = struct { + register: u32, + atom_index: u32, + sym_index: u32, +}; + +/// Used in conjunction with payload to transfer a list of used registers in a compact manner. +pub const RegisterList = struct { + bitset: BitSet = BitSet.initEmpty(), + + const BitSet = IntegerBitSet(32); + const Self = @This(); + + fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt { + for (registers, 0..) |cpreg, i| { + if (reg.id() == cpreg.id()) return @intCast(i); + } + unreachable; // register not in input register list! + } + + pub fn push(self: *Self, registers: []const Register, reg: Register) void { + const index = getIndexForReg(registers, reg); + self.bitset.set(index); + } + + pub fn isSet(self: Self, registers: []const Register, reg: Register) bool { + const index = getIndexForReg(registers, reg); + return self.bitset.isSet(index); + } + + pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) { + return self.bitset.iterator(options); + } + + pub fn count(self: Self) i32 { + return @intCast(self.bitset.count()); + } + + pub fn size(self: Self) i32 { + return @intCast(self.bitset.count() * 8); + } +}; + +const Mir = @This(); +const std = @import("std"); +const builtin = @import("builtin"); +const Type = @import("../../Type.zig"); +const bits = @import("bits.zig"); + +const assert = std.debug.assert; + +const Register = bits.Register; +const CSR = bits.CSR; +const Immediate = bits.Immediate; +const Memory = bits.Memory; +const FrameIndex = bits.FrameIndex; +const FrameAddr = @import("CodeGen.zig").FrameAddr; +const IntegerBitSet = std.bit_set.IntegerBitSet; +const Mnemonic = @import("mnem.zig").Mnemonic; + +const InternPool = @import("../../InternPool.zig"); +const Emit = @import("Emit.zig"); +const codegen = @import("../../codegen.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); diff --git a/src/codegen/riscv64/abi.zig b/src/codegen/riscv64/abi.zig new file mode 100644 index 0000000000000000000000000000000000000000..1a380eab905708b866f18a29762e1653cd0a521b --- /dev/null +++ b/src/codegen/riscv64/abi.zig @@ -0,0 +1,353 @@ +const std = @import("std"); +const bits = @import("bits.zig"); +const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; +const Type = @import("../../Type.zig"); +const InternPool = @import("../../InternPool.zig"); +const Zcu = @import("../../Zcu.zig"); +const assert = std.debug.assert; + +pub const Class = enum { memory, byval, integer, double_integer, fields }; + +pub fn classifyType(ty: Type, zcu: *Zcu) Class { + const target = zcu.getTarget(); + std.debug.assert(ty.hasRuntimeBitsIgnoreComptime(zcu)); + + const max_byval_size = target.ptrBitWidth() * 2; + switch (ty.zigTypeTag(zcu)) { + .@"struct" => { + const bit_size = ty.bitSize(zcu); + if (ty.containerLayout(zcu) == .@"packed") { + if (bit_size > max_byval_size) return .memory; + return .byval; + } + + if (target.cpu.has(.riscv, .d)) fields: { + var any_fp = false; + var field_count: usize = 0; + for (0..ty.structFieldCount(zcu)) |field_index| { + const field_ty = ty.fieldType(field_index, zcu); + if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; + if (field_ty.isRuntimeFloat()) + any_fp = true + else if (!field_ty.isAbiInt(zcu)) + break :fields; + field_count += 1; + if (field_count > 2) break :fields; + } + std.debug.assert(field_count > 0 and field_count <= 2); + if (any_fp) return .fields; + } + + // TODO this doesn't exactly match what clang produces but its better than nothing + if (bit_size > max_byval_size) return .memory; + if (bit_size > max_byval_size / 2) return .double_integer; + return .integer; + }, + .@"union" => { + const bit_size = ty.bitSize(zcu); + if (ty.containerLayout(zcu) == .@"packed") { + if (bit_size > max_byval_size) return .memory; + return .byval; + } + // TODO this doesn't exactly match what clang produces but its better than nothing + if (bit_size > max_byval_size) return .memory; + if (bit_size > max_byval_size / 2) return .double_integer; + return .integer; + }, + .bool => return .integer, + .float => return .byval, + .int, .@"enum", .error_set => { + const bit_size = ty.bitSize(zcu); + if (bit_size > max_byval_size) return .memory; + return .byval; + }, + .vector => { + const bit_size = ty.bitSize(zcu); + if (bit_size > max_byval_size) return .memory; + return .integer; + }, + .optional => { + std.debug.assert(ty.isPtrLikeOptional(zcu)); + return .byval; + }, + .pointer => { + std.debug.assert(!ty.isSlice(zcu)); + return .byval; + }, + .error_union, + .frame, + .@"anyframe", + .noreturn, + .void, + .type, + .comptime_float, + .comptime_int, + .undefined, + .null, + .@"fn", + .@"opaque", + .enum_literal, + .array, + => unreachable, + } +} + +pub const SystemClass = enum { integer, float, memory, none }; + +/// There are a maximum of 8 possible return slots. Returned values are in +/// the beginning of the array; unused slots are filled with .none. +pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass { + var result = [1]SystemClass{.none} ** 8; + const memory_class = [_]SystemClass{ + .memory, .none, .none, .none, + .none, .none, .none, .none, + }; + switch (ty.zigTypeTag(zcu)) { + .bool, .void, .noreturn => { + result[0] = .integer; + return result; + }, + .pointer => switch (ty.ptrSize(zcu)) { + .slice => { + result[0] = .integer; + result[1] = .integer; + return result; + }, + else => { + result[0] = .integer; + return result; + }, + }, + .optional => { + if (ty.isPtrLikeOptional(zcu)) { + result[0] = .integer; + return result; + } + return memory_class; + }, + .int, .@"enum", .error_set => { + const int_bits = ty.intInfo(zcu).bits; + if (int_bits <= 64) { + result[0] = .integer; + return result; + } + if (int_bits <= 128) { + result[0] = .integer; + result[1] = .integer; + return result; + } + unreachable; // support > 128 bit int arguments + }, + .float => { + const target = zcu.getTarget(); + + const float_bits = ty.floatBits(target); + const float_reg_size: u32 = if (target.cpu.has(.riscv, .d)) 64 else 32; + if (float_bits <= float_reg_size) { + result[0] = .float; + return result; + } + unreachable; // support split float args + }, + .error_union => { + const payload_ty = ty.errorUnionPayload(zcu); + const payload_bits = payload_ty.bitSize(zcu); + + // the error union itself + result[0] = .integer; + + // anyerror!void can fit into one register + if (payload_bits == 0) return result; + + return memory_class; + }, + .@"struct", .@"union" => { + const layout = ty.containerLayout(zcu); + const ty_size = ty.abiSize(zcu); + + if (layout == .@"packed") { + assert(ty_size <= 16); + result[0] = .integer; + if (ty_size > 8) result[1] = .integer; + return result; + } + + return memory_class; + }, + .array => { + const ty_size = ty.abiSize(zcu); + if (ty_size <= 8) { + result[0] = .integer; + return result; + } + if (ty_size <= 16) { + result[0] = .integer; + result[1] = .integer; + return result; + } + return memory_class; + }, + .vector => { + // we pass vectors through integer registers if they are small enough to fit. + const vec_bits = ty.totalVectorBits(zcu); + if (vec_bits <= 64) { + result[0] = .integer; + return result; + } + // we should pass vector registers of size <= 128 through 2 integer registers + // but we haven't implemented seperating vector registers into register_pairs + return memory_class; + }, + else => |bad_ty| std.debug.panic("classifySystem {s}", .{@tagName(bad_ty)}), + } +} + +fn classifyStruct( + result: *[8]Class, + byte_offset: *u64, + loaded_struct: InternPool.LoadedStructType, + zcu: *Zcu, +) void { + const ip = &zcu.intern_pool; + var field_it = loaded_struct.iterateRuntimeOrder(ip); + + while (field_it.next()) |field_index| { + const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_align = loaded_struct.fieldAlign(ip, field_index); + byte_offset.* = std.mem.alignForward( + u64, + byte_offset.*, + field_align.toByteUnits() orelse field_ty.abiAlignment(zcu).toByteUnits().?, + ); + if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { + if (field_loaded_struct.layout != .@"packed") { + classifyStruct(result, byte_offset, field_loaded_struct, zcu); + continue; + } + } + const field_class = std.mem.sliceTo(&classifySystem(field_ty, zcu), .none); + const field_size = field_ty.abiSize(zcu); + + combine: { + const result_class = &result[@intCast(byte_offset.* / 8)]; + if (result_class.* == field_class[0]) { + break :combine; + } + + if (result_class.* == .none) { + result_class.* = field_class[0]; + break :combine; + } + assert(field_class[0] != .none); + + // "If one of the classes is MEMORY, the result is the MEMORY class." + if (result_class.* == .memory or field_class[0] == .memory) { + result_class.* = .memory; + break :combine; + } + + // "If one of the classes is INTEGER, the result is the INTEGER." + if (result_class.* == .integer or field_class[0] == .integer) { + result_class.* = .integer; + break :combine; + } + + result_class.* = .integer; + } + @memcpy(result[@intCast(byte_offset.* / 8 + 1)..][0 .. field_class.len - 1], field_class[1..]); + byte_offset.* += field_size; + } +} + +const allocatable_registers = Registers.Integer.all_regs ++ Registers.Float.all_regs ++ Registers.Vector.all_regs; +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); + +const RegisterBitSet = RegisterManager.RegisterBitSet; + +pub const RegisterClass = enum { + int, + float, + vector, +}; + +pub const Registers = struct { + pub const all_preserved = Integer.callee_preserved_regs ++ Float.callee_preserved_regs; + + pub const Integer = struct { + // zig fmt: off + pub const general_purpose = initRegBitSet(0, callee_preserved_regs.len); + pub const function_arg = initRegBitSet(callee_preserved_regs.len, function_arg_regs.len); + pub const function_ret = initRegBitSet(callee_preserved_regs.len, function_ret_regs.len); + pub const temporary = initRegBitSet(callee_preserved_regs.len + function_arg_regs.len, temporary_regs.len); + // zig fmt: on + + pub const callee_preserved_regs = [_]Register{ + // .s0 is omitted to be used as the frame pointer register + .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11, + }; + + pub const function_arg_regs = [_]Register{ + .a0, .a1, .a2, .a3, .a4, .a5, .a6, .a7, + }; + + pub const function_ret_regs = [_]Register{ + .a0, .a1, + }; + + pub const temporary_regs = [_]Register{ + .t0, .t1, .t2, .t3, .t4, .t5, .t6, + }; + + pub const all_regs = callee_preserved_regs ++ function_arg_regs ++ temporary_regs; + }; + + pub const Float = struct { + // zig fmt: off + pub const general_purpose = initRegBitSet(Integer.all_regs.len, callee_preserved_regs.len); + pub const function_arg = initRegBitSet(Integer.all_regs.len + callee_preserved_regs.len, function_arg_regs.len); + pub const function_ret = initRegBitSet(Integer.all_regs.len + callee_preserved_regs.len, function_ret_regs.len); + pub const temporary = initRegBitSet(Integer.all_regs.len + callee_preserved_regs.len + function_arg_regs.len, temporary_regs.len); + // zig fmt: on + + pub const callee_preserved_regs = [_]Register{ + .fs0, .fs1, .fs2, .fs3, .fs4, .fs5, .fs6, .fs7, .fs8, .fs9, .fs10, .fs11, + }; + + pub const function_arg_regs = [_]Register{ + .fa0, .fa1, .fa2, .fa3, .fa4, .fa5, .fa6, .fa7, + }; + + pub const function_ret_regs = [_]Register{ + .fa0, .fa1, + }; + + pub const temporary_regs = [_]Register{ + .ft0, .ft1, .ft2, .ft3, .ft4, .ft5, .ft6, .ft7, .ft8, .ft9, .ft10, .ft11, + }; + + pub const all_regs = callee_preserved_regs ++ function_arg_regs ++ temporary_regs; + }; + + pub const Vector = struct { + pub const general_purpose = initRegBitSet(Integer.all_regs.len + Float.all_regs.len, all_regs.len); + + // zig fmt: off + pub const all_regs = [_]Register{ + .v0, .v1, .v2, .v3, .v4, .v5, .v6, .v7, + .v8, .v9, .v10, .v11, .v12, .v13, .v14, .v15, + .v16, .v17, .v18, .v19, .v20, .v21, .v22, .v23, + .v24, .v25, .v26, .v27, .v28, .v29, .v30, .v31, + }; + // zig fmt: on + }; +}; + +fn initRegBitSet(start: usize, length: usize) RegisterBitSet { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = start, + .end = start + length, + }, true); + return set; +} diff --git a/src/codegen/riscv64/bits.zig b/src/codegen/riscv64/bits.zig new file mode 100644 index 0000000000000000000000000000000000000000..94c64dfc98de50e7ecb8183a2008c7014e222ebd --- /dev/null +++ b/src/codegen/riscv64/bits.zig @@ -0,0 +1,290 @@ +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; +const Target = std.Target; + +const Zcu = @import("../../Zcu.zig"); +const Mir = @import("Mir.zig"); +const abi = @import("abi.zig"); + +pub const Memory = struct { + base: Base, + mod: Mod, + + pub const Base = union(enum) { + reg: Register, + frame: FrameIndex, + }; + + pub const Mod = struct { + size: Size, + unsigned: bool, + disp: i32 = 0, + }; + + pub const Size = enum(u4) { + /// Byte, 1 byte + byte, + /// Half word, 2 bytes + hword, + /// Word, 4 bytes + word, + /// Double word, 8 Bytes + dword, + + pub fn fromByteSize(size: u64) Size { + return switch (size) { + 1...1 => .byte, + 2...2 => .hword, + 3...4 => .word, + 5...8 => .dword, + else => std.debug.panic("fromByteSize {}", .{size}), + }; + } + + pub fn fromBitSize(bit_size: u64) Size { + return switch (bit_size) { + 8 => .byte, + 16 => .hword, + 32 => .word, + 64 => .dword, + else => unreachable, + }; + } + + pub fn bitSize(s: Size) u64 { + return switch (s) { + .byte => 8, + .hword => 16, + .word => 32, + .dword => 64, + }; + } + }; + + /// Asserts `mem` can be represented as a `FrameLoc`. + pub fn toFrameLoc(mem: Memory, mir: Mir) Mir.FrameLoc { + const offset: i32 = mem.mod.disp; + + switch (mem.base) { + .reg => |reg| { + return .{ + .base = reg, + .disp = offset, + }; + }, + .frame => |index| { + const base_loc = mir.frame_locs.get(@intFromEnum(index)); + return .{ + .base = base_loc.base, + .disp = base_loc.disp + offset, + }; + }, + } + } +}; + +pub const Immediate = union(enum) { + signed: i32, + unsigned: u64, + + pub fn u(x: u64) Immediate { + return .{ .unsigned = x }; + } + + pub fn s(x: i32) Immediate { + return .{ .signed = x }; + } + + pub fn asSigned(imm: Immediate, bit_size: u64) i64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(i8, @intCast(x)), + 16 => @as(i16, @intCast(x)), + 32, 64 => x, + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), + 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), + 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), + 64 => @bitCast(x), + else => unreachable, + }, + }; + } + + pub fn asBits(imm: Immediate, comptime T: type) T { + const int_info = @typeInfo(T).int; + if (int_info.signedness != .unsigned) @compileError("Immediate.asBits needs unsigned T"); + return switch (imm) { + .signed => |x| @bitCast(@as(std.meta.Int(.signed, int_info.bits), @intCast(x))), + .unsigned => |x| @intCast(x), + }; + } +}; + +pub const CSR = enum(u12) { + vl = 0xC20, + vtype = 0xC21, + vlenb = 0xC22, +}; + +pub const Register = enum(u8) { + // zig fmt: off + + // base extension registers + + zero, // zero + ra, // return address. caller saved + sp, // stack pointer. callee saved. + gp, // global pointer + tp, // thread pointer + t0, t1, t2, // temporaries. caller saved. + s0, // s0/fp, callee saved. + s1, // callee saved. + a0, a1, // fn args/return values. caller saved. + a2, a3, a4, a5, a6, a7, // fn args. caller saved. + s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, // saved registers. callee saved. + t3, t4, t5, t6, // caller saved + + x0, x1, x2, x3, x4, x5, x6, x7, + x8, x9, x10, x11, x12, x13, x14, x15, + x16, x17, x18, x19, x20, x21, x22, x23, + x24, x25, x26, x27, x28, x29, x30, x31, + + + // F extension registers + + ft0, ft1, ft2, ft3, ft4, ft5, ft6, ft7, // float temporaries. caller saved. + fs0, fs1, // float saved. callee saved. + fa0, fa1, // float arg/ret. caller saved. + fa2, fa3, fa4, fa5, fa6, fa7, // float arg. called saved. + fs2, fs3, fs4, fs5, fs6, fs7, fs8, fs9, fs10, fs11, // float saved. callee saved. + ft8, ft9, ft10, ft11, // foat temporaries. calller saved. + + // this register is accessed only through API instructions instead of directly + // fcsr, + + f0, f1, f2, f3, f4, f5, f6, f7, + f8, f9, f10, f11, f12, f13, f14, f15, + f16, f17, f18, f19, f20, f21, f22, f23, + f24, f25, f26, f27, f28, f29, f30, f31, + + + // V extension registers + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, + + // zig fmt: on + + /// in RISC-V registers are stored as 5 bit IDs and a register can have + /// two names. Example being `zero` and `x0` are the same register and have the + /// same ID, but are two different entries in the enum. We store floating point + /// registers in the same enum. RISC-V uses the same IDs for `f0` and `x0` by + /// infering which register is being talked about given the instruction it's in. + /// + /// The goal of this function is to return the same ID for `zero` and `x0` but two + /// seperate IDs for `x0` and `f0`. We will assume that each register set has 32 registers + /// and is repeated twice, once for the named version, once for the number version. + pub fn id(reg: Register) std.math.IntFittingRange(0, @typeInfo(Register).@"enum".fields.len) { + const base = switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => @intFromEnum(Register.zero), + @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => @intFromEnum(Register.ft0), + @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => @intFromEnum(Register.v0), + else => unreachable, + // zig fmt: on + }; + + return @intCast(base + reg.encodeId()); + } + + pub fn encodeId(reg: Register) u5 { + return @truncate(@intFromEnum(reg)); + } + + pub fn dwarfNum(reg: Register) u8 { + return reg.id(); + } + + pub fn bitSize(reg: Register, zcu: *const Zcu) u32 { + return switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => 64, + @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => if (zcu.getTarget().cpu.has(.riscv, .d)) 64 else 32, + @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => 256, // TODO: look at suggestVectorSize + else => unreachable, + // zig fmt: on + }; + } + + pub fn class(reg: Register) abi.RegisterClass { + return switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => .int, + @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => .float, + @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => .vector, + else => unreachable, + // zig fmt: on + }; + } +}; + +pub const FrameIndex = enum(u32) { + /// This index refers to the return address. + ret_addr, + /// This index refers to the frame pointer. + base_ptr, + /// This index refers to the entire stack frame. + stack_frame, + /// This index referes to where in the stack frame the args are spilled to. + args_frame, + /// This index referes to a frame dedicated to setting up args for function called + /// in this function. Useful for aligning args separately. + call_frame, + /// This index referes to the frame where callee saved registers are spilled and restored from. + spill_frame, + /// Other indices are used for local variable stack slots + _, + + pub const named_count = @typeInfo(FrameIndex).@"enum".fields.len; + + pub fn isNamed(fi: FrameIndex) bool { + return @intFromEnum(fi) < named_count; + } +}; + +/// A linker symbol not yet allocated in VM. +pub const Symbol = struct { + /// Index of the containing atom. + atom_index: u32, + /// Index into the linker's symbol table. + sym_index: u32, +}; + +pub const VType = packed struct(u8) { + vlmul: VlMul, + vsew: VSew, + vta: bool, + vma: bool, +}; + +const VSew = enum(u3) { + @"8" = 0b000, + @"16" = 0b001, + @"32" = 0b010, + @"64" = 0b011, +}; + +const VlMul = enum(u3) { + mf8 = 0b101, + mf4 = 0b110, + mf2 = 0b111, + m1 = 0b000, + m2 = 0b001, + m4 = 0b010, + m8 = 0b011, +}; diff --git a/src/codegen/riscv64/encoding.zig b/src/codegen/riscv64/encoding.zig new file mode 100644 index 0000000000000000000000000000000000000000..109e1d08f8cef5ace0e88a78ecd426536c90f34f --- /dev/null +++ b/src/codegen/riscv64/encoding.zig @@ -0,0 +1,729 @@ +//! This file is responsible for going from MIR, which is emitted by CodeGen +//! and converting it into Instructions, which can be used as needed. +//! +//! Here we encode how mnemonics relate to opcodes and where their operands go. + +/// Lower Instruction Representation +/// +/// This format encodes a specific instruction, however it's still abstracted +/// away from the true encoding it'll be in. It's meant to make the process of +/// indicating unique encoding data easier. +pub const Lir = struct { + opcode: OpCode, + format: Format, + data: Data, + + pub const Format = enum { + R, + I, + S, + B, + U, + J, + extra, + }; + + const Data = union(enum) { + none, + f: struct { funct3: u3 }, + ff: struct { + funct3: u3, + funct7: u7, + }, + sh: struct { + typ: u6, + funct3: u3, + has_5: bool, + }, + + fmt: struct { + funct5: u5, + rm: u3, + fmt: FpFmt, + }, + fcvt: struct { + funct5: u5, + rm: u3, + fmt: FpFmt, + width: Mir.FcvtOp, + }, + + vecls: struct { + width: VecWidth, + umop: Umop, + vm: bool, + mop: Mop, + mew: bool, + nf: u3, + }, + vecmath: struct { + vm: bool, + funct6: u6, + funct3: VecType, + }, + + amo: struct { + funct5: u5, + width: AmoWidth, + }, + fence: struct { + funct3: u3, + fm: FenceMode, + }, + + /// the mnemonic has some special properities that can't be handled in a generic fashion + extra: Mnemonic, + }; + + const OpCode = enum(u7) { + LOAD = 0b0000011, + LOAD_FP = 0b0000111, + MISC_MEM = 0b0001111, + OP_IMM = 0b0010011, + AUIPC = 0b0010111, + OP_IMM_32 = 0b0011011, + STORE = 0b0100011, + STORE_FP = 0b0100111, + AMO = 0b0101111, + OP_V = 0b1010111, + OP = 0b0110011, + OP_32 = 0b0111011, + LUI = 0b0110111, + MADD = 0b1000011, + MSUB = 0b1000111, + NMSUB = 0b1001011, + NMADD = 0b1001111, + OP_FP = 0b1010011, + OP_IMM_64 = 0b1011011, + BRANCH = 0b1100011, + JALR = 0b1100111, + JAL = 0b1101111, + SYSTEM = 0b1110011, + OP_64 = 0b1111011, + NONE = 0b00000000, + }; + + const FpFmt = enum(u2) { + /// 32-bit single-precision + S = 0b00, + /// 64-bit double-precision + D = 0b01, + + // H = 0b10, unused in the G extension + + /// 128-bit quad-precision + Q = 0b11, + }; + + const AmoWidth = enum(u3) { + W = 0b010, + D = 0b011, + }; + + const FenceMode = enum(u4) { + none = 0b0000, + tso = 0b1000, + }; + + const Mop = enum(u2) { + // zig fmt: off + unit = 0b00, + unord = 0b01, + stride = 0b10, + ord = 0b11, + // zig fmt: on + }; + + const Umop = enum(u5) { + // zig fmt: off + unit = 0b00000, + whole = 0b01000, + mask = 0b01011, + fault = 0b10000, + // zig fmt: on + }; + + const VecWidth = enum(u3) { + // zig fmt: off + @"8" = 0b000, + @"16" = 0b101, + @"32" = 0b110, + @"64" = 0b111, + // zig fmt: on + }; + + const VecType = enum(u3) { + OPIVV = 0b000, + OPFVV = 0b001, + OPMVV = 0b010, + OPIVI = 0b011, + OPIVX = 0b100, + OPFVF = 0b101, + OPMVX = 0b110, + }; + + pub fn fromMnem(mnem: Mnemonic) Lir { + return switch (mnem) { + // zig fmt: off + + // OP + .add => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000000 } } }, + .sub => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0100000 } } }, + + .@"and" => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b111, .funct7 = 0b0000000 } } }, + .@"or" => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b110, .funct7 = 0b0000000 } } }, + .xor => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b100, .funct7 = 0b0000000 } } }, + + .sltu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b011, .funct7 = 0b0000000 } } }, + .slt => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b010, .funct7 = 0b0000000 } } }, + + .mul => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000001 } } }, + .mulh => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0000001 } } }, + .mulhsu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b010, .funct7 = 0b0000001 } } }, + .mulhu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b011, .funct7 = 0b0000001 } } }, + + .div => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b100, .funct7 = 0b0000001 } } }, + .divu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000001 } } }, + + .rem => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b110, .funct7 = 0b0000001 } } }, + .remu => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b111, .funct7 = 0b0000001 } } }, + + .sll => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0000000 } } }, + .srl => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000000 } } }, + .sra => .{ .opcode = .OP, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0100000 } } }, + + + // OP_IMM + + .addi => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b000 } } }, + .andi => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b111 } } }, + .xori => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b100 } } }, + + .sltiu => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .f = .{ .funct3 = 0b011 } } }, + + .slli => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b001, .has_5 = true } } }, + .srli => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b101, .has_5 = true } } }, + .srai => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = true } } }, + + .clz => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, + .cpop => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, + + // OP_IMM_32 + + .slliw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b001, .has_5 = false } } }, + .srliw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b101, .has_5 = false } } }, + .sraiw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = false } } }, + + .clzw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, + .cpopw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } }, + + // OP_32 + + .addw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000000 } } }, + .subw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0100000 } } }, + .mulw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b000, .funct7 = 0b0000001 } } }, + + .divw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b100, .funct7 = 0b0000001 } } }, + .divuw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000001 } } }, + + .remw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b110, .funct7 = 0b0000001 } } }, + .remuw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b111, .funct7 = 0b0000001 } } }, + + .sllw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0000000 } } }, + .srlw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0000000 } } }, + .sraw => .{ .opcode = .OP_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b101, .funct7 = 0b0100000 } } }, + + + // OP_FP + + .fadds => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00000, .fmt = .S, .rm = 0b111 } } }, + .faddd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00000, .fmt = .D, .rm = 0b111 } } }, + + .fsubs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00001, .fmt = .S, .rm = 0b111 } } }, + .fsubd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00001, .fmt = .D, .rm = 0b111 } } }, + + .fmuls => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00010, .fmt = .S, .rm = 0b111 } } }, + .fmuld => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00010, .fmt = .D, .rm = 0b111 } } }, + + .fdivs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00011, .fmt = .S, .rm = 0b111 } } }, + .fdivd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00011, .fmt = .D, .rm = 0b111 } } }, + + .fmins => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .S, .rm = 0b000 } } }, + .fmind => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .D, .rm = 0b000 } } }, + + .fmaxs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .S, .rm = 0b001 } } }, + .fmaxd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00101, .fmt = .D, .rm = 0b001 } } }, + + .fsqrts => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b01011, .fmt = .S, .rm = 0b111 } } }, + .fsqrtd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b01011, .fmt = .D, .rm = 0b111 } } }, + + .fles => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .S, .rm = 0b000 } } }, + .fled => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .D, .rm = 0b000 } } }, + + .flts => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .S, .rm = 0b001 } } }, + .fltd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .D, .rm = 0b001 } } }, + + .feqs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .S, .rm = 0b010 } } }, + .feqd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b10100, .fmt = .D, .rm = 0b010 } } }, + + .fsgnjns => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .S, .rm = 0b000 } } }, + .fsgnjnd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .D, .rm = 0b000 } } }, + + .fsgnjxs => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .S, .rm = 0b010 } } }, + .fsgnjxd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fmt = .{ .funct5 = 0b00100, .fmt = .D, .rm = 0b010 } } }, + + .fcvtws => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .w } } }, + .fcvtwus => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .wu } } }, + .fcvtls => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .l } } }, + .fcvtlus => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .S, .rm = 0b111, .width = .lu } } }, + + .fcvtwd => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .w } } }, + .fcvtwud => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .wu } } }, + .fcvtld => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .l } } }, + .fcvtlud => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11000, .fmt = .D, .rm = 0b111, .width = .lu } } }, + + .fcvtsw => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .w } } }, + .fcvtswu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .wu } } }, + .fcvtsl => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .l } } }, + .fcvtslu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .S, .rm = 0b111, .width = .lu } } }, + + .fcvtdw => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .w } } }, + .fcvtdwu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .wu } } }, + .fcvtdl => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .l } } }, + .fcvtdlu => .{ .opcode = .OP_FP, .format = .R, .data = .{ .fcvt = .{ .funct5 = 0b11010, .fmt = .D, .rm = 0b111, .width = .lu } } }, + + // LOAD + + .lb => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b000 } } }, + .lh => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b001 } } }, + .lw => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b010 } } }, + .ld => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b011 } } }, + .lbu => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b100 } } }, + .lhu => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b101 } } }, + .lwu => .{ .opcode = .LOAD, .format = .I, .data = .{ .f = .{ .funct3 = 0b110 } } }, + + + // STORE + + .sb => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b000 } } }, + .sh => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b001 } } }, + .sw => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b010 } } }, + .sd => .{ .opcode = .STORE, .format = .S, .data = .{ .f = .{ .funct3 = 0b011 } } }, + + + // LOAD_FP + + .flw => .{ .opcode = .LOAD_FP, .format = .I, .data = .{ .f = .{ .funct3 = 0b010 } } }, + .fld => .{ .opcode = .LOAD_FP, .format = .I, .data = .{ .f = .{ .funct3 = 0b011 } } }, + + .vle8v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"8", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + .vle16v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"16", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + .vle32v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"32", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + .vle64v => .{ .opcode = .LOAD_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"64", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + + + // STORE_FP + + .fsw => .{ .opcode = .STORE_FP, .format = .S, .data = .{ .f = .{ .funct3 = 0b010 } } }, + .fsd => .{ .opcode = .STORE_FP, .format = .S, .data = .{ .f = .{ .funct3 = 0b011 } } }, + + .vse8v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"8", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + .vse16v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"16", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + .vse32v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"32", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + .vse64v => .{ .opcode = .STORE_FP, .format = .R, .data = .{ .vecls = .{ .width = .@"64", .umop = .unit, .vm = true, .mop = .unit, .mew = false, .nf = 0b000 } } }, + + // JALR + + .jalr => .{ .opcode = .JALR, .format = .I, .data = .{ .f = .{ .funct3 = 0b000 } } }, + + + // LUI + + .lui => .{ .opcode = .LUI, .format = .U, .data = .{ .none = {} } }, + + + // AUIPC + + .auipc => .{ .opcode = .AUIPC, .format = .U, .data = .{ .none = {} } }, + + + // JAL + + .jal => .{ .opcode = .JAL, .format = .J, .data = .{ .none = {} } }, + + + // BRANCH + + .beq => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b000 } } }, + .bne => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b001 } } }, + + + // SYSTEM + + .ecall => .{ .opcode = .SYSTEM, .format = .extra, .data = .{ .extra = .ecall } }, + .ebreak => .{ .opcode = .SYSTEM, .format = .extra, .data = .{ .extra = .ebreak } }, + + .csrrs => .{ .opcode = .SYSTEM, .format = .I, .data = .{ .f = .{ .funct3 = 0b010 } } }, + + + // NONE + + .unimp => .{ .opcode = .NONE, .format = .extra, .data = .{ .extra = .unimp } }, + + + // MISC_MEM + + .fence => .{ .opcode = .MISC_MEM, .format = .I, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .none } } }, + .fencetso => .{ .opcode = .MISC_MEM, .format = .I, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .tso } } }, + + + // AMO + + .amoaddw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } }, + .amoswapw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } }, + .lrw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00010 } } }, + .scw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00011 } } }, + .amoxorw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } }, + .amoandw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } }, + .amoorw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } }, + .amominw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10000 } } }, + .amomaxw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10100 } } }, + .amominuw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } }, + .amomaxuw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } }, + + + .amoaddd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } }, + .amoswapd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } }, + .lrd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00010 } } }, + .scd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00011 } } }, + .amoxord => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } }, + .amoandd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } }, + .amoord => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } }, + .amomind => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10000 } } }, + .amomaxd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10100 } } }, + .amominud => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11000 } } }, + .amomaxud => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } }, + + // OP_V + .vsetivli => .{ .opcode = .OP_V, .format = .I, .data = .{ .f = .{ .funct3 = 0b111 } } }, + .vsetvli => .{ .opcode = .OP_V, .format = .I, .data = .{ .f = .{ .funct3 = 0b111 } } }, + .vaddvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000000, .funct3 = .OPIVV } } }, + .vsubvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000010, .funct3 = .OPIVV } } }, + .vmulvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b100101, .funct3 = .OPIVV } } }, + + .vfaddvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000000, .funct3 = .OPFVV } } }, + .vfsubvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000010, .funct3 = .OPFVV } } }, + .vfmulvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b100100, .funct3 = .OPFVV } } }, + + .vadcvv => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b010000, .funct3 = .OPMVV } } }, + .vmvvx => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b010111, .funct3 = .OPIVX } } }, + + .vslidedownvx => .{ .opcode = .OP_V, .format = .R, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b001111, .funct3 = .OPIVX } } }, + + + .pseudo_prologue, + .pseudo_epilogue, + .pseudo_dbg_prologue_end, + .pseudo_dbg_epilogue_begin, + .pseudo_dbg_line_column, + .pseudo_load_rm, + .pseudo_store_rm, + .pseudo_lea_rm, + .pseudo_j, + .pseudo_dead, + .pseudo_load_symbol, + .pseudo_load_tlv, + .pseudo_mv, + .pseudo_restore_regs, + .pseudo_spill_regs, + .pseudo_compare, + .pseudo_not, + .pseudo_extern_fn_reloc, + .nop, + => std.debug.panic("lir: didn't catch pseudo {s}", .{@tagName(mnem)}), + // zig fmt: on + }; + } +}; + +/// This is the final form of the instruction. Lir is transformed into +/// this, which is then bitcast into a u32. +pub const Instruction = union(Lir.Format) { + R: packed struct(u32) { + opcode: u7, + rd: u5, + funct3: u3, + rs1: u5, + rs2: u5, + funct7: u7, + }, + I: packed struct(u32) { + opcode: u7, + rd: u5, + funct3: u3, + rs1: u5, + imm0_11: u12, + }, + S: packed struct(u32) { + opcode: u7, + imm0_4: u5, + funct3: u3, + rs1: u5, + rs2: u5, + imm5_11: u7, + }, + B: packed struct(u32) { + opcode: u7, + imm11: u1, + imm1_4: u4, + funct3: u3, + rs1: u5, + rs2: u5, + imm5_10: u6, + imm12: u1, + }, + U: packed struct(u32) { + opcode: u7, + rd: u5, + imm12_31: u20, + }, + J: packed struct(u32) { + opcode: u7, + rd: u5, + imm12_19: u8, + imm11: u1, + imm1_10: u10, + imm20: u1, + }, + extra: u32, + + comptime { + for (std.meta.fields(Instruction)) |field| { + assert(@bitSizeOf(field.type) == 32); + } + } + + pub const Operand = union(enum) { + none, + reg: Register, + csr: CSR, + mem: Memory, + imm: Immediate, + barrier: Mir.Barrier, + }; + + pub fn toU32(inst: Instruction) u32 { + return switch (inst) { + inline else => |v| @bitCast(v), + }; + } + + pub fn encode(inst: Instruction, writer: *std.Io.Writer) !void { + try writer.writeInt(u32, inst.toU32(), .little); + } + + pub fn fromLir(lir: Lir, ops: []const Operand) Instruction { + const opcode: u7 = @intFromEnum(lir.opcode); + + switch (lir.format) { + .R => { + return .{ + .R = switch (lir.data) { + .ff => |ff| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .rs2 = ops[2].reg.encodeId(), + + .opcode = opcode, + .funct3 = ff.funct3, + .funct7 = ff.funct7, + }, + .fmt => |fmt| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .rs2 = ops[2].reg.encodeId(), + + .opcode = opcode, + .funct3 = fmt.rm, + .funct7 = (@as(u7, fmt.funct5) << 2) | @intFromEnum(fmt.fmt), + }, + .fcvt => |fcvt| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .rs2 = @intFromEnum(fcvt.width), + + .opcode = opcode, + .funct3 = fcvt.rm, + .funct7 = (@as(u7, fcvt.funct5) << 2) | @intFromEnum(fcvt.fmt), + }, + .vecls => |vec| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + + .rs2 = @intFromEnum(vec.umop), + + .opcode = opcode, + .funct3 = @intFromEnum(vec.width), + .funct7 = (@as(u7, vec.nf) << 4) | (@as(u7, @intFromBool(vec.mew)) << 3) | (@as(u7, @intFromEnum(vec.mop)) << 1) | @intFromBool(vec.vm), + }, + .vecmath => |vec| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .rs2 = ops[2].reg.encodeId(), + + .opcode = opcode, + .funct3 = @intFromEnum(vec.funct3), + .funct7 = (@as(u7, vec.funct6) << 1) | @intFromBool(vec.vm), + }, + .amo => |amo| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .rs2 = ops[2].reg.encodeId(), + + .opcode = opcode, + .funct3 = @intFromEnum(amo.width), + .funct7 = @as(u7, amo.funct5) << 2 | + @as(u7, @intFromBool(ops[3].barrier == .rl)) << 1 | + @as(u7, @intFromBool(ops[4].barrier == .aq)), + }, + else => unreachable, + }, + }; + }, + .S => { + assert(ops.len == 3); + const umm = ops[2].imm.asBits(u12); + return .{ + .S = .{ + .imm0_4 = @truncate(umm), + .rs1 = ops[0].reg.encodeId(), + .rs2 = ops[1].reg.encodeId(), + .imm5_11 = @truncate(umm >> 5), + + .opcode = opcode, + .funct3 = lir.data.f.funct3, + }, + }; + }, + .I => { + return .{ + .I = switch (lir.data) { + .f => |f| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .imm0_11 = ops[2].imm.asBits(u12), + + .opcode = opcode, + .funct3 = f.funct3, + }, + .sh => |sh| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .imm0_11 = (@as(u12, sh.typ) << 6) | + if (sh.has_5) ops[2].imm.asBits(u6) else (@as(u6, 0) | ops[2].imm.asBits(u5)), + + .opcode = opcode, + .funct3 = sh.funct3, + }, + .fence => |fence| .{ + .rd = 0, + .rs1 = 0, + .funct3 = 0, + .imm0_11 = (@as(u12, @intFromEnum(fence.fm)) << 8) | + (@as(u12, @intFromEnum(ops[1].barrier)) << 4) | + @as(u12, @intFromEnum(ops[0].barrier)), + .opcode = opcode, + }, + else => unreachable, + }, + }; + }, + .U => { + assert(ops.len == 2); + return .{ + .U = .{ + .rd = ops[0].reg.encodeId(), + .imm12_31 = ops[1].imm.asBits(u20), + + .opcode = opcode, + }, + }; + }, + .J => { + assert(ops.len == 2); + + const umm = ops[1].imm.asBits(u21); + // the RISC-V spec says the target index of a jump + // must be a multiple of 2 + assert(umm % 2 == 0); + + return .{ + .J = .{ + .rd = ops[0].reg.encodeId(), + .imm1_10 = @truncate(umm >> 1), + .imm11 = @truncate(umm >> 11), + .imm12_19 = @truncate(umm >> 12), + .imm20 = @truncate(umm >> 20), + + .opcode = opcode, + }, + }; + }, + .B => { + assert(ops.len == 3); + + const umm = ops[2].imm.asBits(u13); + // the RISC-V spec says the target index of a branch + // must be a multiple of 2 + assert(umm % 2 == 0); + + return .{ + .B = .{ + .rs1 = ops[0].reg.encodeId(), + .rs2 = ops[1].reg.encodeId(), + .imm1_4 = @truncate(umm >> 1), + .imm5_10 = @truncate(umm >> 5), + .imm11 = @truncate(umm >> 11), + .imm12 = @truncate(umm >> 12), + + .opcode = opcode, + .funct3 = lir.data.f.funct3, + }, + }; + }, + .extra => { + assert(ops.len == 0); + + return .{ + .I = .{ + .rd = Register.zero.encodeId(), + .rs1 = Register.zero.encodeId(), + .imm0_11 = switch (lir.data.extra) { + .ecall => 0x000, + .ebreak => 0x001, + .unimp => 0x000, + else => unreachable, + }, + + .opcode = opcode, + .funct3 = 0b000, + }, + }; + }, + } + } +}; + +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.format); + +const bits = @import("bits.zig"); +const Mir = @import("Mir.zig"); +const Mnemonic = @import("mnem.zig").Mnemonic; +const Lower = @import("Lower.zig"); + +const Register = bits.Register; +const CSR = bits.CSR; +const Memory = bits.Memory; +const Immediate = bits.Immediate; diff --git a/src/codegen/riscv64/mnem.zig b/src/codegen/riscv64/mnem.zig new file mode 100644 index 0000000000000000000000000000000000000000..cb4734e59cf7769a4f1efefb572d1252e1e1ad30 --- /dev/null +++ b/src/codegen/riscv64/mnem.zig @@ -0,0 +1,257 @@ +pub const Mnemonic = enum(u16) { + // Arithmetics + addi, + add, + addw, + + sub, + subw, + + // Bits + xori, + xor, + @"or", + + @"and", + andi, + + slt, + sltu, + sltiu, + + slli, + srli, + srai, + + slliw, + srliw, + sraiw, + + sll, + srl, + sra, + + sllw, + srlw, + sraw, + + // Control Flow + jalr, + jal, + + beq, + bne, + + // Memory + lui, + auipc, + + ld, + lw, + lh, + lb, + lbu, + lhu, + lwu, + + sd, + sw, + sh, + sb, + + // System + ebreak, + ecall, + unimp, + nop, + + // M extension + mul, + mulh, + mulhu, + mulhsu, + mulw, + + div, + divu, + divw, + divuw, + + rem, + remu, + remw, + remuw, + + // F extension (32-bit float) + fadds, + fsubs, + fmuls, + fdivs, + + fmins, + fmaxs, + + fsqrts, + + flw, + fsw, + + feqs, + flts, + fles, + + // D extension (64-bit float) + faddd, + fsubd, + fmuld, + fdivd, + + fmind, + fmaxd, + + fsqrtd, + + fld, + fsd, + + feqd, + fltd, + fled, + + fcvtws, + fcvtwus, + fcvtls, + fcvtlus, + + fcvtwd, + fcvtwud, + fcvtld, + fcvtlud, + + fcvtsw, + fcvtswu, + fcvtsl, + fcvtslu, + + fcvtdw, + fcvtdwu, + fcvtdl, + fcvtdlu, + + fsgnjns, + fsgnjnd, + + fsgnjxs, + fsgnjxd, + + // Zicsr Extension Instructions + csrrs, + + // V Extension Instructions + vsetvli, + vsetivli, + vaddvv, + vfaddvv, + vsubvv, + vfsubvv, + vmulvv, + vfmulvv, + vslidedownvx, + + vle8v, + vle16v, + vle32v, + vle64v, + + vse8v, + vse16v, + vse32v, + vse64v, + + vadcvv, + vmvvx, + + // Zbb Extension Instructions + clz, + clzw, + cpop, + cpopw, + + // A Extension Instructions + fence, + fencetso, + + lrw, + scw, + amoswapw, + amoaddw, + amoandw, + amoorw, + amoxorw, + amomaxw, + amominw, + amomaxuw, + amominuw, + + lrd, + scd, + amoswapd, + amoaddd, + amoandd, + amoord, + amoxord, + amomaxd, + amomind, + amomaxud, + amominud, + + // Pseudo-instructions. Used for anything that isn't 1:1 with an + // assembly instruction. + + /// Pseudo-instruction that will generate a backpatched + /// function prologue. + pseudo_prologue, + /// Pseudo-instruction that will generate a backpatched + /// function epilogue + pseudo_epilogue, + + /// Pseudo-instruction: End of prologue + pseudo_dbg_prologue_end, + /// Pseudo-instruction: Beginning of epilogue + pseudo_dbg_epilogue_begin, + /// Pseudo-instruction: Update debug line + pseudo_dbg_line_column, + + /// Pseudo-instruction that loads from memory into a register. + pseudo_load_rm, + /// Pseudo-instruction that stores from a register into memory + pseudo_store_rm, + /// Pseudo-instruction that loads the address of memory into a register. + pseudo_lea_rm, + /// Jumps. Uses `inst` payload. + pseudo_j, + /// Dead inst, ignored by the emitter. + pseudo_dead, + /// Loads the address of a value that hasn't yet been allocated in memory. + pseudo_load_symbol, + /// Loads the address of a TLV. + pseudo_load_tlv, + + /// Moves the value of rs1 to rd. + pseudo_mv, + + pseudo_restore_regs, + pseudo_spill_regs, + + pseudo_compare, + pseudo_not, + pseudo_extern_fn_reloc, +}; + +pub const Pseudo = enum(u8) { + li, + mv, + tail, + beqz, + ret, +}; diff --git a/src/codegen/sparc64/CodeGen.zig b/src/codegen/sparc64/CodeGen.zig new file mode 100644 index 0000000000000000000000000000000000000000..48d44e39f973c4f1b92604a34ee6258186b425a2 --- /dev/null +++ b/src/codegen/sparc64/CodeGen.zig @@ -0,0 +1,4827 @@ +//! SPARCv9 codegen. +//! This lowers AIR into MIR. +//! For now this only implements medium/low code model with absolute addressing. +//! TODO add support for other code models. +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.codegen); +const math = std.math; +const mem = std.mem; +const Allocator = mem.Allocator; +const builtin = @import("builtin"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); +const InternPool = @import("../../InternPool.zig"); +const Value = @import("../../Value.zig"); +const ErrorMsg = Zcu.ErrorMsg; +const codegen = @import("../../codegen.zig"); +const Air = @import("../../Air.zig"); +const Mir = @import("Mir.zig"); +const Emit = @import("Emit.zig"); +const Type = @import("../../Type.zig"); +const CodeGenError = codegen.CodeGenError; +const Endian = std.builtin.Endian; +const Alignment = InternPool.Alignment; + +const build_options = @import("build_options"); + +const bits = @import("bits.zig"); +const abi = @import("abi.zig"); +const errUnionPayloadOffset = codegen.errUnionPayloadOffset; +const errUnionErrorOffset = codegen.errUnionErrorOffset; +const Instruction = bits.Instruction; +const ASI = Instruction.ASI; +const ShiftWidth = Instruction.ShiftWidth; +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; +const Register = bits.Register; +const gp = abi.RegisterClass.gp; + +const Self = @This(); + +const InnerError = CodeGenError || error{OutOfRegisters}; + +pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { + return null; +} + +const RegisterView = enum(u1) { + caller, + callee, +}; + +gpa: Allocator, +pt: Zcu.PerThread, +air: Air, +liveness: Air.Liveness, +bin_file: *link.File, +target: *const std.Target, +func_index: InternPool.Index, +err_msg: ?*ErrorMsg, +args: []MCValue, +ret_mcv: MCValue, +fn_type: Type, +arg_index: usize, +src_loc: Zcu.LazySrcLoc, +stack_align: Alignment, + +/// MIR Instructions +mir_instructions: std.MultiArrayList(Mir.Inst) = .{}, +/// MIR extra data +mir_extra: std.ArrayListUnmanaged(u32) = .empty, + +/// Byte offset within the source file of the ending curly. +end_di_line: u32, +end_di_column: u32, + +/// The value is an offset into the `Function` `code` from the beginning. +/// To perform the reloc, write 32-bit signed little-endian integer +/// which is a relative jump, based on the address following the reloc. +exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, + +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, + +/// Whenever there is a runtime branch, we push a Branch onto this stack, +/// and pop it off when the runtime branch joins. This provides an "overlay" +/// of the table of mappings from instructions to `MCValue` from within the branch. +/// This way we can modify the `MCValue` for an instruction in different ways +/// within different branches. Special consideration is needed when a branch +/// joins with its parent, to make sure all instructions have the same MCValue +/// across each runtime branch upon joining. +branch_stack: *std.array_list.Managed(Branch), + +// Key is the block instruction +blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, + +register_manager: RegisterManager = .{}, + +/// Maps offset to what is stored there. +stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .empty, + +/// Tracks the current instruction allocated to the condition flags +condition_flags_inst: ?Air.Inst.Index = null, + +/// Tracks the current instruction allocated to the condition register +condition_register_inst: ?Air.Inst.Index = null, + +/// Offset from the stack base, representing the end of the stack frame. +max_end_stack: u32 = 0, +/// Represents the current end stack offset. If there is no existing slot +/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`. +next_stack_offset: u32 = 0, + +/// Debug field, used to find bugs in the compiler. +air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init, + +const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; + +const MCValue = union(enum) { + /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. + /// TODO Look into deleting this tag and using `dead` instead, since every use + /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. + none, + /// Control flow will not allow this value to be observed. + unreach, + /// No more references to this value remain. + dead, + /// The value is undefined. + undef, + /// A pointer-sized integer that fits in a register. + /// If the type is a pointer, this is the pointer address in virtual address space. + immediate: u64, + /// The value is in a target-specific register. + register: Register, + /// The value is a tuple { wrapped, overflow } where + /// wrapped is stored in the register and the overflow bit is + /// stored in the C (signed) or V (unsigned) flag of the CCR. + /// + /// This MCValue is only generated by a add_with_overflow or + /// sub_with_overflow instruction operating on 32- or 64-bit values. + register_with_overflow: struct { + reg: Register, + flag: struct { cond: Instruction.ICondition, ccr: Instruction.CCR }, + }, + /// The value is in memory at a hard-coded address. + /// If the type is a pointer, it means the pointer address is at this memory location. + memory: u64, + /// The value is one of the stack variables. + /// If the type is a pointer, it means the pointer address is in the stack at this offset. + /// Note that this stores the plain value (i.e without the effects of the stack bias). + /// Always convert this value into machine offsets with realStackOffset() before + /// lowering into asm! + stack_offset: u32, + /// The value is a pointer to one of the stack variables (payload is stack offset). + ptr_stack_offset: u32, + /// The value is in the specified CCR. The value is 1 (if + /// the type is u1) or true (if the type in bool) iff the + /// specified condition is true. + condition_flags: struct { + cond: Instruction.Condition, + ccr: Instruction.CCR, + }, + /// The value is in the specified Register. The value is 1 (if + /// the type is u1) or true (if the type in bool) iff the + /// specified condition is true. + condition_register: struct { + cond: Instruction.RCondition, + reg: Register, + }, + + fn isMemory(mcv: MCValue) bool { + return switch (mcv) { + .memory, .stack_offset => true, + else => false, + }; + } + + fn isImmediate(mcv: MCValue) bool { + return switch (mcv) { + .immediate => true, + else => false, + }; + } + + fn isMutable(mcv: MCValue) bool { + return switch (mcv) { + .none => unreachable, + .unreach => unreachable, + .dead => unreachable, + + .immediate, + .memory, + .condition_flags, + .condition_register, + .ptr_stack_offset, + .undef, + => false, + + .register, + .stack_offset, + => true, + }; + } +}; + +const Branch = struct { + inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty, + + fn deinit(self: *Branch, gpa: Allocator) void { + self.inst_table.deinit(gpa); + self.* = undefined; + } +}; + +const StackAllocation = struct { + inst: Air.Inst.Index, + /// TODO do we need size? should be determined by inst.ty.abiSize() + size: u32, +}; + +const BlockData = struct { + relocs: std.ArrayListUnmanaged(Mir.Inst.Index), + /// The first break instruction encounters `null` here and chooses a + /// machine code value for the block result, populating this field. + /// Following break instructions encounter that value and use it for + /// the location to store their block results. + mcv: MCValue, +}; + +const CallMCValues = struct { + args: []MCValue, + return_value: MCValue, + stack_byte_count: u32, + stack_align: Alignment, + + fn deinit(self: *CallMCValues, func: *Self) void { + func.gpa.free(self.args); + self.* = undefined; + } +}; + +const BigTomb = struct { + function: *Self, + inst: Air.Inst.Index, + lbt: Air.Liveness.BigTomb, + + fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { + const dies = bt.lbt.feed(); + const op_index = op_ref.toIndex() orelse return; + if (!dies) return; + bt.function.processDeath(op_index); + } + + fn finishAir(bt: *BigTomb, result: MCValue) void { + const is_used = !bt.function.liveness.isUnused(bt.inst); + if (is_used) { + log.debug("%{d} => {}", .{ bt.inst, result }); + const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1]; + branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result); + } + bt.function.finishAirBookkeeping(); + } +}; + +pub fn generate( + lf: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + air: *const Air, + liveness: *const ?Air.Liveness, +) CodeGenError!Mir { + const zcu = pt.zcu; + const gpa = zcu.gpa; + const func = zcu.funcInfo(func_index); + const func_ty = Type.fromInterned(func.ty); + const file_scope = zcu.navFileScope(func.owner_nav); + const target = &file_scope.mod.?.resolved_target.result; + + var branch_stack = std.array_list.Managed(Branch).init(gpa); + defer { + assert(branch_stack.items.len == 1); + branch_stack.items[0].deinit(gpa); + branch_stack.deinit(); + } + try branch_stack.append(.{}); + + var function: Self = .{ + .gpa = gpa, + .pt = pt, + .air = air.*, + .liveness = liveness.*.?, + .target = target, + .bin_file = lf, + .func_index = func_index, + .err_msg = null, + .args = undefined, // populated after `resolveCallingConventionValues` + .ret_mcv = undefined, // populated after `resolveCallingConventionValues` + .fn_type = func_ty, + .arg_index = 0, + .branch_stack = &branch_stack, + .src_loc = src_loc, + .stack_align = undefined, + .end_di_line = func.rbrace_line, + .end_di_column = func.rbrace_column, + }; + defer function.stack.deinit(gpa); + defer function.blocks.deinit(gpa); + defer function.exitlude_jump_relocs.deinit(gpa); + + var call_info = function.resolveCallingConventionValues(func_ty, .callee) catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + else => |e| return e, + }; + defer call_info.deinit(&function); + + function.args = call_info.args; + function.ret_mcv = call_info.return_value; + function.stack_align = call_info.stack_align; + function.max_end_stack = call_info.stack_byte_count; + + function.gen() catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), + else => |e| return e, + }; + + var mir: Mir = .{ + .instructions = function.mir_instructions.toOwnedSlice(), + .extra = &.{}, // fallible, so populated after errdefer + }; + errdefer mir.deinit(gpa); + mir.extra = try function.mir_extra.toOwnedSlice(gpa); + return mir; +} + +fn gen(self: *Self) !void { + const pt = self.pt; + const zcu = pt.zcu; + const cc = self.fn_type.fnCallingConvention(zcu); + if (cc != .naked) { + // TODO Finish function prologue and epilogue for sparc64. + + // save %sp, stack_reserved_area, %sp + const save_inst = try self.addInst(.{ + .tag = .save, + .data = .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = .sp, + .rs1 = .sp, + .rs2_or_imm = .{ .imm = -abi.stack_reserved_area }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .dbg_prologue_end, + .data = .{ .nop = {} }, + }); + + try self.genBody(self.air.getMainBody()); + + _ = try self.addInst(.{ + .tag = .dbg_epilogue_begin, + .data = .{ .nop = {} }, + }); + + // exitlude jumps + if (self.exitlude_jump_relocs.items.len > 0 and + self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 3) + { + // If the last Mir instruction (apart from the + // dbg_epilogue_begin) is the last exitlude jump + // relocation (which would just jump two instructions + // further), it can be safely removed + const index = self.exitlude_jump_relocs.pop().?; + + // First, remove the delay slot, then remove + // the branch instruction itself. + self.mir_instructions.orderedRemove(index + 1); + self.mir_instructions.orderedRemove(index); + } + + for (self.exitlude_jump_relocs.items) |jmp_reloc| { + self.mir_instructions.set(jmp_reloc, .{ + .tag = .bpcc, + .data = .{ + .branch_predict_int = .{ + .ccr = .xcc, + .cond = .al, + .inst = @as(u32, @intCast(self.mir_instructions.len)), + }, + }, + }); + } + + // Backpatch stack offset + const total_stack_size = self.max_end_stack + abi.stack_reserved_area; + const stack_size = self.stack_align.forward(total_stack_size); + if (math.cast(i13, stack_size)) |size| { + self.mir_instructions.set(save_inst, .{ + .tag = .save, + .data = .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = .sp, + .rs1 = .sp, + .rs2_or_imm = .{ .imm = -size }, + }, + }, + }); + } else { + // TODO for large stacks, replace the prologue with: + // setx stack_size, %g1 + // save %sp, %g1, %sp + return self.fail("TODO SPARCv9: allow larger stacks", .{}); + } + + // return %i7 + 8 + _ = try self.addInst(.{ + .tag = .@"return", + .data = .{ + .arithmetic_2op = .{ + .is_imm = true, + .rs1 = .i7, + .rs2_or_imm = .{ .imm = 8 }, + }, + }, + }); + + // Branches in SPARC have a delay slot, that is, the instruction + // following it will unconditionally be executed. + // See: Section 3.2.3 Control Transfer in SPARCv9 manual. + // See also: https://arcb.csc.ncsu.edu/~mueller/codeopt/codeopt00/notes/delaybra.html + // TODO Find a way to fill this delay slot + // nop + _ = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); + } else { + _ = try self.addInst(.{ + .tag = .dbg_prologue_end, + .data = .{ .nop = {} }, + }); + + try self.genBody(self.air.getMainBody()); + + _ = try self.addInst(.{ + .tag = .dbg_epilogue_begin, + .data = .{ .nop = {} }, + }); + } + + // Drop them off at the rbrace. + _ = try self.addInst(.{ + .tag = .dbg_line, + .data = .{ .dbg_line_column = .{ + .line = self.end_di_line, + .column = self.end_di_column, + } }, + }); +} + +fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const air_tags = self.air.instructions.items(.tag); + + for (body) |inst| { + // TODO: remove now-redundant isUnused calls from AIR handler functions + if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) + continue; + + const old_air_bookkeeping = self.air_bookkeeping; + try self.ensureProcessDeathCapacity(Air.Liveness.bpi); + + self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); + switch (air_tags[@intFromEnum(inst)]) { + // zig fmt: off + .ptr_add => try self.airPtrArithmetic(inst, .ptr_add), + .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub), + + .add => try self.airBinOp(inst, .add), + .add_wrap => try self.airBinOp(inst, .add_wrap), + .sub => try self.airBinOp(inst, .sub), + .sub_wrap => try self.airBinOp(inst, .sub_wrap), + .mul => try self.airBinOp(inst, .mul), + .mul_wrap => try self.airBinOp(inst, .mul_wrap), + .shl => try self.airBinOp(inst, .shl), + .shl_exact => try self.airBinOp(inst, .shl_exact), + .shr => try self.airBinOp(inst, .shr), + .shr_exact => try self.airBinOp(inst, .shr_exact), + .bool_and => try self.airBinOp(inst, .bool_and), + .bool_or => try self.airBinOp(inst, .bool_or), + .bit_and => try self.airBinOp(inst, .bit_and), + .bit_or => try self.airBinOp(inst, .bit_or), + .xor => try self.airBinOp(inst, .xor), + + .add_sat => try self.airAddSat(inst), + .sub_sat => try self.airSubSat(inst), + .mul_sat => try self.airMulSat(inst), + .shl_sat => try self.airShlSat(inst), + .min, .max => try self.airMinMax(inst), + .rem => try self.airRem(inst), + .mod => try self.airMod(inst), + .slice => try self.airSlice(inst), + + .sqrt, + .sin, + .cos, + .tan, + .exp, + .exp2, + .log, + .log2, + .log10, + .abs, + .floor, + .ceil, + .round, + .trunc_float, + .neg, + => try self.airUnaryMath(inst), + + .add_with_overflow => try self.airAddSubWithOverflow(inst), + .sub_with_overflow => try self.airAddSubWithOverflow(inst), + .mul_with_overflow => try self.airMulWithOverflow(inst), + .shl_with_overflow => try self.airShlWithOverflow(inst), + + .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst), + + .cmp_lt => try self.airCmp(inst, .lt), + .cmp_lte => try self.airCmp(inst, .lte), + .cmp_eq => try self.airCmp(inst, .eq), + .cmp_gte => try self.airCmp(inst, .gte), + .cmp_gt => try self.airCmp(inst, .gt), + .cmp_neq => try self.airCmp(inst, .neq), + .cmp_vector => @panic("TODO try self.airCmpVector(inst)"), + .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), + + .alloc => try self.airAlloc(inst), + .ret_ptr => try self.airRetPtr(inst), + .arg => try self.airArg(inst), + .assembly => try self.airAsm(inst), + .bitcast => try self.airBitCast(inst), + .block => try self.airBlock(inst), + .br => try self.airBr(inst), + .repeat => return self.fail("TODO implement `repeat`", .{}), + .switch_dispatch => return self.fail("TODO implement `switch_dispatch`", .{}), + .trap => try self.airTrap(), + .breakpoint => try self.airBreakpoint(), + .ret_addr => @panic("TODO try self.airRetAddr(inst)"), + .frame_addr => @panic("TODO try self.airFrameAddress(inst)"), + .cond_br => try self.airCondBr(inst), + .fptrunc => @panic("TODO try self.airFptrunc(inst)"), + .fpext => @panic("TODO try self.airFpext(inst)"), + .intcast => try self.airIntCast(inst), + .trunc => try self.airTrunc(inst), + .is_non_null => try self.airIsNonNull(inst), + .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"), + .is_null => try self.airIsNull(inst), + .is_null_ptr => @panic("TODO try self.airIsNullPtr(inst)"), + .is_non_err => try self.airIsNonErr(inst), + .is_non_err_ptr => @panic("TODO try self.airIsNonErrPtr(inst)"), + .is_err => try self.airIsErr(inst), + .is_err_ptr => @panic("TODO try self.airIsErrPtr(inst)"), + .load => try self.airLoad(inst), + .loop => try self.airLoop(inst), + .not => try self.airNot(inst), + .ret => try self.airRet(inst), + .ret_safe => try self.airRet(inst), // TODO + .ret_load => try self.airRetLoad(inst), + .store => try self.airStore(inst, false), + .store_safe => try self.airStore(inst, true), + .struct_field_ptr=> try self.airStructFieldPtr(inst), + .struct_field_val=> try self.airStructFieldVal(inst), + .array_to_slice => try self.airArrayToSlice(inst), + .float_from_int => try self.airFloatFromInt(inst), + .int_from_float => try self.airIntFromFloat(inst), + .cmpxchg_strong, + .cmpxchg_weak, + => try self.airCmpxchg(inst), + .atomic_rmw => try self.airAtomicRmw(inst), + .atomic_load => try self.airAtomicLoad(inst), + .memcpy => @panic("TODO try self.airMemcpy(inst)"), + .memmove => @panic("TODO try self.airMemmove(inst)"), + .memset => try self.airMemset(inst, false), + .memset_safe => try self.airMemset(inst, true), + .set_union_tag => try self.airSetUnionTag(inst), + .get_union_tag => try self.airGetUnionTag(inst), + .clz => try self.airClz(inst), + .ctz => try self.airCtz(inst), + .popcount => try self.airPopcount(inst), + .byte_swap => try self.airByteSwap(inst), + .bit_reverse => try self.airBitReverse(inst), + .tag_name => try self.airTagName(inst), + .error_name => try self.airErrorName(inst), + .splat => try self.airSplat(inst), + .select => @panic("TODO try self.airSelect(inst)"), + .shuffle_one => @panic("TODO try self.airShuffleOne(inst)"), + .shuffle_two => @panic("TODO try self.airShuffleTwo(inst)"), + .reduce => @panic("TODO try self.airReduce(inst)"), + .aggregate_init => try self.airAggregateInit(inst), + .union_init => try self.airUnionInit(inst), + .prefetch => try self.airPrefetch(inst), + .mul_add => @panic("TODO try self.airMulAdd(inst)"), + .addrspace_cast => @panic("TODO try self.airAddrSpaceCast(int)"), + + .@"try" => try self.airTry(inst), + .try_cold => try self.airTry(inst), + .try_ptr => @panic("TODO try self.airTryPtr(inst)"), + .try_ptr_cold => @panic("TODO try self.airTryPtrCold(inst)"), + + .dbg_stmt => try self.airDbgStmt(inst), + .dbg_empty_stmt => self.finishAirBookkeeping(), + .dbg_inline_block => try self.airDbgInlineBlock(inst), + .dbg_var_ptr, + .dbg_var_val, + .dbg_arg_inline, + => try self.airDbgVar(inst), + + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + + .atomic_store_unordered => @panic("TODO try self.airAtomicStore(inst, .unordered)"), + .atomic_store_monotonic => @panic("TODO try self.airAtomicStore(inst, .monotonic)"), + .atomic_store_release => @panic("TODO try self.airAtomicStore(inst, .release)"), + .atomic_store_seq_cst => @panic("TODO try self.airAtomicStore(inst, .seq_cst)"), + + .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), + .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), + .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), + .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), + + .field_parent_ptr => @panic("TODO try self.airFieldParentPtr(inst)"), + + .switch_br => try self.airSwitch(inst), + .loop_switch_br => return self.fail("TODO implement `loop_switch_br`", .{}), + .slice_ptr => try self.airSlicePtr(inst), + .slice_len => try self.airSliceLen(inst), + + .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst), + .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst), + + .array_elem_val => try self.airArrayElemVal(inst), + .slice_elem_val => try self.airSliceElemVal(inst), + .slice_elem_ptr => @panic("TODO try self.airSliceElemPtr(inst)"), + .ptr_elem_val => try self.airPtrElemVal(inst), + .ptr_elem_ptr => try self.airPtrElemPtr(inst), + + .inferred_alloc, .inferred_alloc_comptime => unreachable, + .unreach => self.finishAirBookkeeping(), + + .optional_payload => try self.airOptionalPayload(inst), + .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), + .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), + .unwrap_errunion_err => try self.airUnwrapErrErr(inst), + .unwrap_errunion_payload => try self.airUnwrapErrPayload(inst), + .unwrap_errunion_err_ptr => @panic("TODO try self.airUnwrapErrErrPtr(inst)"), + .unwrap_errunion_payload_ptr=> @panic("TODO try self.airUnwrapErrPayloadPtr(inst)"), + .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), + .err_return_trace => @panic("TODO try self.airErrReturnTrace(inst)"), + .set_err_return_trace => @panic("TODO try self.airSetErrReturnTrace(inst)"), + .save_err_return_trace_index=> @panic("TODO try self.airSaveErrReturnTraceIndex(inst)"), + + .wrap_optional => try self.airWrapOptional(inst), + .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst), + .wrap_errunion_err => try self.airWrapErrUnionErr(inst), + + .add_optimized, + .sub_optimized, + .mul_optimized, + .div_float_optimized, + .div_trunc_optimized, + .div_floor_optimized, + .div_exact_optimized, + .rem_optimized, + .mod_optimized, + .neg_optimized, + .cmp_lt_optimized, + .cmp_lte_optimized, + .cmp_eq_optimized, + .cmp_gte_optimized, + .cmp_gt_optimized, + .cmp_neq_optimized, + .cmp_vector_optimized, + .reduce_optimized, + .int_from_float_optimized, + => @panic("TODO implement optimized float mode"), + + .add_safe, + .sub_safe, + .mul_safe, + .intcast_safe, + .int_from_float_safe, + .int_from_float_optimized_safe, + => @panic("TODO implement safety_checked_instructions"), + + .is_named_enum_value => @panic("TODO implement is_named_enum_value"), + .error_set_has_value => @panic("TODO implement error_set_has_value"), + .vector_store_elem => @panic("TODO implement vector_store_elem"), + .runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"), + + .c_va_arg => return self.fail("TODO implement c_va_arg", .{}), + .c_va_copy => return self.fail("TODO implement c_va_copy", .{}), + .c_va_end => return self.fail("TODO implement c_va_end", .{}), + .c_va_start => return self.fail("TODO implement c_va_start", .{}), + + .wasm_memory_size => unreachable, + .wasm_memory_grow => unreachable, + + .work_item_id => unreachable, + .work_group_size => unreachable, + .work_group_id => unreachable, + // zig fmt: on + } + + assert(!self.register_manager.lockedRegsExist()); + + if (std.debug.runtime_safety) { + if (self.air_bookkeeping < old_air_bookkeeping + 1) { + std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{t}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] }); + } + } + } +} + +fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { + const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + const pt = self.pt; + const zcu = pt.zcu; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const lhs = try self.resolveInst(extra.lhs); + const rhs = try self.resolveInst(extra.rhs); + const lhs_ty = self.typeOf(extra.lhs); + const rhs_ty = self.typeOf(extra.rhs); + + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}), + .int => { + assert(lhs_ty.eql(rhs_ty, zcu)); + const int_info = lhs_ty.intInfo(zcu); + switch (int_info.bits) { + 32, 64 => { + // Only say yes if the operation is + // commutative, i.e. we can swap both of the + // operands + const lhs_immediate_ok = switch (tag) { + .add_with_overflow => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .sub_with_overflow => false, + else => unreachable, + }; + const rhs_immediate_ok = switch (tag) { + .add_with_overflow, + .sub_with_overflow, + => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), + else => unreachable, + }; + + const mir_tag: Mir.Inst.Tag = switch (tag) { + .add_with_overflow => .addcc, + .sub_with_overflow => .subcc, + else => unreachable, + }; + + try self.spillConditionFlagsIfOccupied(); + + const dest = blk: { + if (rhs_immediate_ok) { + break :blk try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, null); + } else if (lhs_immediate_ok) { + // swap lhs and rhs + break :blk try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, null); + } else { + break :blk try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, null); + } + }; + + const cond = switch (int_info.signedness) { + .unsigned => switch (tag) { + .add_with_overflow => Instruction.ICondition.cs, + .sub_with_overflow => Instruction.ICondition.cc, + else => unreachable, + }, + .signed => Instruction.ICondition.vs, + }; + + const ccr = switch (int_info.bits) { + 32 => Instruction.CCR.icc, + 64 => Instruction.CCR.xcc, + else => unreachable, + }; + + break :result MCValue{ .register_with_overflow = .{ + .reg = dest.register, + .flag = .{ .cond = cond, .ccr = ccr }, + } }; + }, + else => return self.fail("TODO overflow operations on other integer sizes", .{}), + } + }, + else => unreachable, + } + }; + return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const vector_ty = self.typeOfIndex(inst); + const len = vector_ty.vectorLen(zcu); + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); + const result: MCValue = res: { + if (self.liveness.isUnused(inst)) break :res MCValue.dead; + return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch}); + }; + + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); + @memcpy(buf[0..elements.len], elements); + return self.finishAir(inst, result, buf); + } + var bt = try self.iterateBigTomb(inst, elements.len); + for (elements) |elem| { + bt.feed(elem); + } + return bt.finishAir(result); +} + +fn airAlloc(self: *Self, inst: Air.Inst.Index) !void { + const stack_offset = try self.allocMemPtr(inst); + return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none }); +} + +fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const ptr_ty = self.typeOf(ty_op.operand); + const ptr = try self.resolveInst(ty_op.operand); + const array_ty = ptr_ty.childType(zcu); + const array_len: u32 = @intCast(array_ty.arrayLen(zcu)); + const ptr_bytes = 8; + const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8"); + try self.genSetStack(ptr_ty, stack_offset, ptr); + try self.genSetStack(Type.usize, stack_offset - ptr_bytes, .{ .immediate = array_len }); + break :result MCValue{ .stack_offset = stack_offset }; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airAsm(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Asm, ty_pl.payload); + const is_volatile = extra.data.flags.is_volatile; + const outputs_len = extra.data.flags.outputs_len; + var extra_i: usize = extra.end; + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + outputs_len]); + extra_i += outputs.len; + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.inputs_len]); + extra_i += inputs.len; + + const dead = !is_volatile and self.liveness.isUnused(inst); + const result: MCValue = if (dead) .dead else result: { + if (outputs.len > 1) { + return self.fail("TODO implement codegen for asm with more than 1 output", .{}); + } + + const output_constraint: ?[]const u8 = for (outputs) |output| { + if (output != .none) { + return self.fail("TODO implement codegen for non-expr asm", .{}); + } + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + break constraint; + } else null; + + for (inputs) |input| { + const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(input_bytes, 0); + const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') { + return self.fail("unrecognized asm input constraint: '{s}'", .{constraint}); + } + const reg_name = constraint[1 .. constraint.len - 1]; + const reg = parseRegName(reg_name) orelse + return self.fail("unrecognized register: '{s}'", .{reg_name}); + + const arg_mcv = try self.resolveInst(input); + try self.register_manager.getReg(reg, null); + try self.genSetReg(self.typeOf(input), reg, arg_mcv); + } + + // TODO honor the clobbers + _ = extra.data.clobbers; + + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; + + if (mem.eql(u8, asm_source, "ta 0x6d")) { + _ = try self.addInst(.{ + .tag = .tcc, + .data = .{ + .trap = .{ + .is_imm = true, + .cond = .al, + .rs2_or_imm = .{ .imm = 0x6d }, + }, + }, + }); + } else { + return self.fail("TODO implement a full SPARCv9 assembly parsing", .{}); + } + + if (output_constraint) |output| { + if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { + return self.fail("unrecognized asm output constraint: '{s}'", .{output}); + } + const reg_name = output[2 .. output.len - 1]; + const reg = parseRegName(reg_name) orelse + return self.fail("unrecognized register: '{s}'", .{reg_name}); + break :result MCValue{ .register = reg }; + } else { + break :result MCValue{ .none = {} }; + } + }; + + simple: { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); + var buf_index: usize = 0; + for (outputs) |output| { + if (output == .none) continue; + + if (buf_index >= buf.len) break :simple; + buf[buf_index] = output; + buf_index += 1; + } + if (buf_index + inputs.len > buf.len) break :simple; + @memcpy(buf[buf_index..][0..inputs.len], inputs); + return self.finishAir(inst, result, buf); + } + + var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len); + for (outputs) |output| { + if (output == .none) continue; + + bt.feed(output); + } + for (inputs) |input| { + bt.feed(input); + } + return bt.finishAir(result); +} + +fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const arg_index = self.arg_index; + self.arg_index += 1; + + const ty = self.typeOfIndex(inst); + const mcv: MCValue = blk: { + switch (self.args[arg_index]) { + .stack_offset => |off| { + const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse { + return self.fail("type '{f}' too big to fit into stack frame", .{ty.fmt(pt)}); + }; + const offset = off + abi_size; + break :blk .{ .stack_offset = offset }; + }, + else => |mcv| break :blk mcv, + } + }; + + const func_zir = zcu.funcInfo(self.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?; + const file = zcu.fileByIndex(func_zir.file); + if (!file.mod.?.strip) { + const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg; + const zir = &file.zir.?; + const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); + + self.genArgDbgInfo(name, ty, mcv) catch |err| + return self.fail("failed to generate debug info for parameter: {s}", .{@errorName(err)}); + } + + if (self.liveness.isUnused(inst)) + return self.finishAirBookkeeping(); + + switch (mcv) { + .register => |reg| { + self.register_manager.getRegAssumeFree(reg, inst); + }, + else => {}, + } + + return self.finishAir(inst, mcv, .{ .none, .none, .none }); +} + +fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void { + _ = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; + + return self.fail("TODO implement airAtomicLoad for {}", .{ + self.target.cpu.arch, + }); +} + +fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { + _ = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + + return self.fail("TODO implement airAtomicRmw for {}", .{ + self.target.cpu.arch, + }); +} + +fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + const result: MCValue = if (self.liveness.isUnused(inst)) + .dead + else + try self.binOp(tag, lhs, rhs, lhs_ty, rhs_ty, BinOpMetadata{ + .lhs = bin_op.lhs, + .rhs = bin_op.rhs, + .inst = inst, + }); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + const result: MCValue = if (self.liveness.isUnused(inst)) + .dead + else + try self.binOp(tag, lhs, rhs, lhs_ty, rhs_ty, BinOpMetadata{ + .lhs = bin_op.lhs, + .rhs = bin_op.rhs, + .inst = inst, + }); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(ty_op.operand); + if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand; + + const operand_lock = switch (operand) { + .register => |reg| self.register_manager.lockReg(reg), + .register_with_overflow => |rwo| self.register_manager.lockReg(rwo.reg), + else => null, + }; + defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); + + const dest = try self.allocRegOrMem(inst, true); + try self.setRegOrMem(self.typeOfIndex(inst), dest, operand); + break :result dest; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airBlock(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Block, ty_pl.payload); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); +} + +fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { + try self.blocks.putNoClobber(self.gpa, inst, .{ + // A block is a setup to be able to jump to the end. + .relocs = .{}, + // It also acts as a receptacle for break operands. + // Here we use `MCValue.none` to represent a null value so that the first + // break instruction will choose a MCValue for the block result and overwrite + // this field. Following break instructions will use that MCValue to put their + // block results. + .mcv = MCValue{ .none = {} }, + }); + defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa); + + // TODO emit debug info lexical block + try self.genBody(body); + + // relocations for `bpcc` instructions + const relocs = &self.blocks.getPtr(inst).?.relocs; + if (relocs.items.len > 0 and relocs.items[relocs.items.len - 1] == self.mir_instructions.len - 1) { + // If the last Mir instruction is the last relocation (which + // would just jump two instruction further), it can be safely + // removed + const index = relocs.pop().?; + + // First, remove the delay slot, then remove + // the branch instruction itself. + self.mir_instructions.orderedRemove(index + 1); + self.mir_instructions.orderedRemove(index); + } + for (relocs.items) |reloc| { + try self.performReloc(reloc); + } + + const result = self.blocks.getPtr(inst).?.mcv; + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airBr(self: *Self, inst: Air.Inst.Index) !void { + const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; + try self.br(branch.block_inst, branch.operand); + return self.finishAir(inst, .dead, .{ branch.operand, .none, .none }); +} + +fn airTrap(self: *Self) !void { + // ta 0x05 + _ = try self.addInst(.{ + .tag = .tcc, + .data = .{ + .trap = .{ + .is_imm = true, + .cond = .al, + .rs2_or_imm = .{ .imm = 0x05 }, + }, + }, + }); + return self.finishAirBookkeeping(); +} + +fn airBreakpoint(self: *Self) !void { + // ta 0x01 + _ = try self.addInst(.{ + .tag = .tcc, + .data = .{ + .trap = .{ + .is_imm = true, + .cond = .al, + .rs2_or_imm = .{ .imm = 0x01 }, + }, + }, + }); + return self.finishAirBookkeeping(); +} + +fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + // We have hardware byteswapper in SPARCv9, don't let mainstream compilers mislead you. + // That being said, the strategy to lower this is: + // - If src is an immediate, comptime-swap it. + // - If src is in memory then issue an LD*A with #ASI_P_[oppposite-endian] + // - If src is a register then issue an ST*A with #ASI_P_[oppposite-endian] + // to a stack slot, then follow with a normal load from said stack slot. + // This is because on some implementations, ASI-tagged memory operations are non-piplelinable + // and loads tend to have longer latency than stores, so the sequence will minimize stall. + // The result will always be either another immediate or stored in a register. + // TODO: Fold byteswap+store into a single ST*A and load+byteswap into a single LD*A. + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + switch (operand_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO byteswap for vectors", .{}), + .int => { + const int_info = operand_ty.intInfo(zcu); + if (int_info.bits == 8) break :result operand; + + const abi_size = int_info.bits >> 3; + const abi_align = operand_ty.abiAlignment(zcu); + const opposite_endian_asi = switch (self.target.cpu.arch.endian()) { + Endian.big => ASI.asi_primary_little, + Endian.little => ASI.asi_primary, + }; + + switch (operand) { + .immediate => |imm| { + const swapped = switch (int_info.bits) { + 16 => @byteSwap(@as(u16, @intCast(imm))), + 24 => @byteSwap(@as(u24, @intCast(imm))), + 32 => @byteSwap(@as(u32, @intCast(imm))), + 40 => @byteSwap(@as(u40, @intCast(imm))), + 48 => @byteSwap(@as(u48, @intCast(imm))), + 56 => @byteSwap(@as(u56, @intCast(imm))), + 64 => @byteSwap(@as(u64, @intCast(imm))), + else => return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}), + }; + break :result .{ .immediate = swapped }; + }, + .register => |reg| { + if (int_info.bits > 64 or @popCount(int_info.bits) != 1) + return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}); + + const off = try self.allocMem(inst, abi_size, abi_align); + const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) }); + + try self.genStoreASI(reg, .sp, off_reg, abi_size, opposite_endian_asi); + try self.genLoad(reg, .sp, Register, off_reg, abi_size); + break :result .{ .register = reg }; + }, + .memory => { + if (int_info.bits > 64 or @popCount(int_info.bits) != 1) + return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}); + + const addr_reg = try self.copyToTmpRegister(operand_ty, operand); + const dst_reg = try self.register_manager.allocReg(null, gp); + + try self.genLoadASI(dst_reg, addr_reg, .g0, abi_size, opposite_endian_asi); + break :result .{ .register = dst_reg }; + }, + .stack_offset => |off| { + if (int_info.bits > 64 or @popCount(int_info.bits) != 1) + return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}); + + const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) }); + const dst_reg = try self.register_manager.allocReg(null, gp); + + try self.genLoadASI(dst_reg, .sp, off_reg, abi_size, opposite_endian_asi); + break :result .{ .register = dst_reg }; + }, + else => unreachable, + } + }, + else => unreachable, + } + }; + + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void { + if (modifier == .always_tail) return self.fail("TODO implement tail calls for {}", .{self.target.cpu.arch}); + + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const callee = pl_op.operand; + const extra = self.air.extraData(Air.Call, pl_op.payload); + const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end .. extra.end + extra.data.args_len]); + const ty = self.typeOf(callee); + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const fn_ty = switch (ty.zigTypeTag(zcu)) { + .@"fn" => ty, + .pointer => ty.childType(zcu), + else => unreachable, + }; + + var info = try self.resolveCallingConventionValues(fn_ty, .caller); + defer info.deinit(self); + + // CCR is volatile across function calls + // (SCD 2.4.1, page 3P-10) + try self.spillConditionFlagsIfOccupied(); + + // Save caller-saved registers, but crucially *after* we save the + // compare flags as saving compare flags may require a new + // caller-saved register + for (abi.caller_preserved_regs) |reg| { + try self.register_manager.getReg(reg, null); + } + + for (info.args, 0..) |mc_arg, arg_i| { + const arg = args[arg_i]; + const arg_ty = self.typeOf(arg); + const arg_mcv = try self.resolveInst(arg); + + switch (mc_arg) { + .none => continue, + .register => |reg| { + try self.register_manager.getReg(reg, null); + try self.genSetReg(arg_ty, reg, arg_mcv); + }, + .stack_offset => { + return self.fail("TODO implement calling with parameters in memory", .{}); + }, + .ptr_stack_offset => { + return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); + }, + else => unreachable, + } + } + + // Due to incremental compilation, how function calls are generated depends + // on linking. + if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { + .func => { + return self.fail("TODO implement calling functions", .{}); + }, + .@"extern" => { + return self.fail("TODO implement calling extern functions", .{}); + }, + else => { + return self.fail("TODO implement calling bitcasted functions", .{}); + }, + } else { + assert(ty.zigTypeTag(zcu) == .pointer); + const mcv = try self.resolveInst(callee); + try self.genSetReg(ty, .o7, mcv); + + _ = try self.addInst(.{ + .tag = .jmpl, + .data = .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = .o7, + .rs1 = .o7, + .rs2_or_imm = .{ .rs2 = .g0 }, + }, + }, + }); + + // TODO Find a way to fill this delay slot + _ = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); + } + + const result = info.return_value; + + if (args.len + 1 <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); + buf[0] = callee; + @memcpy(buf[1..][0..args.len], args); + return self.finishAir(inst, result, buf); + } + + var bt = try self.iterateBigTomb(inst, 1 + args.len); + bt.feed(callee); + for (args) |arg| { + bt.feed(arg); + } + return bt.finishAir(result); +} + +fn airClz(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const pt = self.pt; + const zcu = pt.zcu; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.typeOf(bin_op.lhs); + + const int_ty = switch (lhs_ty.zigTypeTag(zcu)) { + .vector => unreachable, // Handled by cmp_vector. + .@"enum" => lhs_ty.intTagType(zcu), + .int => lhs_ty, + .bool => Type.u1, + .pointer => Type.usize, + .error_set => Type.u16, + .optional => blk: { + const payload_ty = lhs_ty.optionalChild(zcu); + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :blk Type.u1; + } else if (lhs_ty.isPtrLikeOptional(zcu)) { + break :blk Type.usize; + } else { + return self.fail("TODO SPARCv9 cmp non-pointer optionals", .{}); + } + }, + .float => return self.fail("TODO SPARCv9 cmp floats", .{}), + else => unreachable, + }; + + const int_info = int_ty.intInfo(zcu); + if (int_info.bits <= 64) { + _ = try self.binOp(.cmp_eq, lhs, rhs, int_ty, int_ty, BinOpMetadata{ + .lhs = bin_op.lhs, + .rhs = bin_op.rhs, + .inst = inst, + }); + + try self.spillConditionFlagsIfOccupied(); + self.condition_flags_inst = inst; + + break :result switch (int_info.signedness) { + .signed => MCValue{ .condition_flags = .{ + .cond = .{ .icond = Instruction.ICondition.fromCompareOperatorSigned(op) }, + .ccr = .xcc, + } }, + .unsigned => MCValue{ .condition_flags = .{ + .cond = .{ .icond = Instruction.ICondition.fromCompareOperatorUnsigned(op) }, + .ccr = .xcc, + } }, + }; + } else { + return self.fail("TODO SPARCv9 cmp for ints > 64 bits", .{}); + } + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + _ = operand; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Block, ty_pl.payload); + _ = extra; + + return self.fail("TODO implement airCmpxchg for {}", .{ + self.target.cpu.arch, + }); +} + +fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const condition = try self.resolveInst(pl_op.operand); + const extra = self.air.extraData(Air.CondBr, pl_op.payload); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const liveness_condbr = self.liveness.getCondBr(inst); + + // Here we emit a branch to the false section. + const reloc: Mir.Inst.Index = try self.condBr(condition); + + // If the condition dies here in this condbr instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + if (self.liveness.operandDies(inst, 0)) { + if (pl_op.operand.toIndex()) |op_index| { + self.processDeath(op_index); + } + } + + // Capture the state of register and stack allocation state so that we can revert to it. + const parent_next_stack_offset = self.next_stack_offset; + const parent_free_registers = self.register_manager.free_registers; + var parent_stack = try self.stack.clone(self.gpa); + defer parent_stack.deinit(self.gpa); + const parent_registers = self.register_manager.registers; + const parent_condition_flags_inst = self.condition_flags_inst; + + try self.branch_stack.append(.{}); + errdefer { + _ = self.branch_stack.pop().?; + } + + try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len); + for (liveness_condbr.then_deaths) |operand| { + self.processDeath(operand); + } + try self.genBody(then_body); + + // Revert to the previous register and stack allocation state. + + var saved_then_branch = self.branch_stack.pop().?; + defer saved_then_branch.deinit(self.gpa); + + self.register_manager.registers = parent_registers; + self.condition_flags_inst = parent_condition_flags_inst; + + self.stack.deinit(self.gpa); + self.stack = parent_stack; + parent_stack = .{}; + + self.next_stack_offset = parent_next_stack_offset; + self.register_manager.free_registers = parent_free_registers; + + try self.performReloc(reloc); + const else_branch = self.branch_stack.addOneAssumeCapacity(); + else_branch.* = .{}; + + try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len); + for (liveness_condbr.else_deaths) |operand| { + self.processDeath(operand); + } + try self.genBody(else_body); + + // At this point, each branch will possibly have conflicting values for where + // each instruction is stored. They agree, however, on which instructions are alive/dead. + // We use the first ("then") branch as canonical, and here emit + // instructions into the second ("else") branch to make it conform. + // We continue respect the data structure semantic guarantees of the else_branch so + // that we can use all the code emitting abstractions. This is why at the bottom we + // assert that parent_branch.free_registers equals the saved_then_branch.free_registers + // rather than assigning it. + const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2]; + try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count()); + + const else_slice = else_branch.inst_table.entries.slice(); + const else_keys = else_slice.items(.key); + const else_values = else_slice.items(.value); + for (else_keys, 0..) |else_key, else_idx| { + const else_value = else_values[else_idx]; + const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) |then_entry| blk: { + // The instruction's MCValue is overridden in both branches. + log.debug("condBr put branch table (key = %{d}, value = {})", .{ else_key, then_entry.value }); + parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value); + if (else_value == .dead) { + assert(then_entry.value == .dead); + continue; + } + break :blk then_entry.value; + } else blk: { + if (else_value == .dead) + continue; + // The instruction is only overridden in the else branch. + var i: usize = self.branch_stack.items.len - 2; + while (true) { + i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead? + if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| { + assert(mcv != .dead); + break :blk mcv; + } + } + }; + log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv }); + // TODO make sure the destination stack offset / register does not already have something + // going on there. + try self.setRegOrMem(self.typeOfIndex(else_key), canon_mcv, else_value); + // TODO track the new register / stack allocation + } + try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count()); + const then_slice = saved_then_branch.inst_table.entries.slice(); + const then_keys = then_slice.items(.key); + const then_values = then_slice.items(.value); + for (then_keys, 0..) |then_key, then_idx| { + const then_value = then_values[then_idx]; + // We already deleted the items from this table that matched the else_branch. + // So these are all instructions that are only overridden in the then branch. + parent_branch.inst_table.putAssumeCapacity(then_key, then_value); + if (then_value == .dead) + continue; + const parent_mcv = blk: { + var i: usize = self.branch_stack.items.len - 2; + while (true) { + i -= 1; + if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| { + assert(mcv != .dead); + break :blk mcv; + } + } + }; + log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value }); + // TODO make sure the destination stack offset / register does not already have something + // going on there. + try self.setRegOrMem(self.typeOfIndex(then_key), parent_mcv, then_value); + // TODO track the new register / stack allocation + } + + { + var item = self.branch_stack.pop().?; + item.deinit(self.gpa); + } + + // We already took care of pl_op.operand earlier, so we're going + // to pass .none here + return self.finishAir(inst, .unreach, .{ .none, .none, .none }); +} + +fn airCtz(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + // TODO emit debug info for function change + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); +} + +fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void { + const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; + + _ = try self.addInst(.{ + .tag = .dbg_line, + .data = .{ + .dbg_line_column = .{ + .line = dbg_stmt.line, + .column = dbg_stmt.column, + }, + }, + }); + + return self.finishAirBookkeeping(); +} + +fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); + const operand = pl_op.operand; + // TODO emit debug info for this variable + _ = name; + return self.finishAir(inst, .dead, .{ operand, .none, .none }); +} + +fn airDiv(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airErrorName(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else { + _ = operand; + return self.fail("TODO implement airErrorName for {}", .{self.target.cpu.arch}); + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntFromFloat for {}", .{ + self.target.cpu.arch, + }); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + if (self.liveness.isUnused(inst)) + return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); + + const pt = self.pt; + const zcu = pt.zcu; + const operand_ty = self.typeOf(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); + const info_a = operand_ty.intInfo(zcu); + const info_b = self.typeOfIndex(inst).intInfo(zcu); + if (info_a.signedness != info_b.signedness) + return self.fail("TODO gen intcast sign safety in semantic analysis", .{}); + + if (info_a.bits == info_b.bits) + return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none }); + + return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch}); +} + +fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatFromInt for {}", .{ + self.target.cpu.arch, + }); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIsErr(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(un_op); + const ty = self.typeOf(un_op); + break :result try self.isErr(ty, operand); + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(un_op); + const ty = self.typeOf(un_op); + break :result try self.isNonErr(ty, operand); + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airIsNull(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(un_op); + break :result try self.isNull(operand); + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(un_op); + break :result try self.isNonNull(operand); + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airLoad(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const elem_ty = self.typeOfIndex(inst); + const elem_size = elem_ty.abiSize(zcu); + const result: MCValue = result: { + if (!elem_ty.hasRuntimeBits(zcu)) + break :result MCValue.none; + + const ptr = try self.resolveInst(ty_op.operand); + const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(zcu); + if (self.liveness.isUnused(inst) and !is_volatile) + break :result MCValue.dead; + + const dst_mcv: MCValue = blk: { + if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) { + // The MCValue that holds the pointer can be re-used as the value. + break :blk switch (ptr) { + .register => |r| MCValue{ .register = r }, + else => ptr, + }; + } else { + break :blk try self.allocRegOrMem(inst, true); + } + }; + try self.load(dst_mcv, ptr, self.typeOf(ty_op.operand)); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airLoop(self: *Self, inst: Air.Inst.Index) !void { + // A loop is a setup to be able to jump back to the beginning. + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const loop = self.air.extraData(Air.Block, ty_pl.payload); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end .. loop.end + loop.data.body_len]); + const start: u32 = @intCast(self.mir_instructions.len); + + try self.genBody(body); + try self.jump(start); + + return self.finishAirBookkeeping(); +} + +fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { + if (safety) { + // TODO if the value is undef, write 0xaa bytes to dest + } else { + // TODO if the value is undef, don't lower this instruction + } + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload); + + const operand = pl_op.operand; + const value = extra.data.lhs; + const length = extra.data.rhs; + _ = operand; + _ = value; + _ = length; + + return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch}); +} + +fn airMinMax(self: *Self, inst: Air.Inst.Index) !void { + const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + + const result: MCValue = if (self.liveness.isUnused(inst)) + .dead + else + try self.minMax(tag, lhs, rhs, lhs_ty, rhs_ty); + + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMod(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + assert(lhs_ty.eql(rhs_ty, self.pt.zcu)); + + if (self.liveness.isUnused(inst)) + return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); + + // TODO add safety check + + // We use manual assembly emission to generate faster code + // First, ensure lhs, rhs, rem, and added are in registers + + const lhs_is_register = lhs == .register; + const rhs_is_register = rhs == .register; + + const lhs_reg = if (lhs_is_register) + lhs.register + else + try self.register_manager.allocReg(null, gp); + + const lhs_lock = self.register_manager.lockReg(lhs_reg); + defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); + + const rhs_reg = if (rhs_is_register) + rhs.register + else + try self.register_manager.allocReg(null, gp); + const rhs_lock = self.register_manager.lockReg(rhs_reg); + defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); + + if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); + if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); + + const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp); + const regs_locks = self.register_manager.lockRegsAssumeUnused(2, regs); + defer for (regs_locks) |reg| { + self.register_manager.unlockReg(reg); + }; + + const add_reg = regs[0]; + const mod_reg = regs[1]; + + // mod_reg = @rem(lhs_reg, rhs_reg) + _ = try self.addInst(.{ + .tag = .sdivx, + .data = .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = mod_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .mulx, + .data = .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = mod_reg, + .rs1 = mod_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .sub, + .data = .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = mod_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .rs2 = mod_reg }, + }, + }, + }); + + // add_reg = mod_reg + rhs_reg + _ = try self.addInst(.{ + .tag = .add, + .data = .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = add_reg, + .rs1 = mod_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + }); + + // if (add_reg == rhs_reg) add_reg = 0 + _ = try self.addInst(.{ + .tag = .cmp, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = add_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .movcc, + .data = .{ + .conditional_move_int = .{ + .is_imm = true, + .ccr = .xcc, + .cond = .{ .icond = .eq }, + .rd = add_reg, + .rs2_or_imm = .{ .imm = 0 }, + }, + }, + }); + + // if (lhs_reg < 0) mod_reg = add_reg + _ = try self.addInst(.{ + .tag = .movr, + .data = .{ + .conditional_move_reg = .{ + .is_imm = false, + .cond = .lt_zero, + .rd = mod_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .rs2 = add_reg }, + }, + }, + }); + + return self.finishAir(inst, .{ .register = mod_reg }, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { + //const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + const pt = self.pt; + const zcu = pt.zcu; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const lhs = try self.resolveInst(extra.lhs); + const rhs = try self.resolveInst(extra.rhs); + const lhs_ty = self.typeOf(extra.lhs); + const rhs_ty = self.typeOf(extra.rhs); + + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO implement mul_with_overflow for vectors", .{}), + .int => { + assert(lhs_ty.eql(rhs_ty, zcu)); + const int_info = lhs_ty.intInfo(zcu); + switch (int_info.bits) { + 1...32 => { + try self.spillConditionFlagsIfOccupied(); + + const dest = try self.binOp(.mul, lhs, rhs, lhs_ty, rhs_ty, null); + + const dest_reg = dest.register; + const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); + defer self.register_manager.unlockReg(dest_reg_lock); + + const truncated_reg = try self.register_manager.allocReg(null, gp); + const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); + defer self.register_manager.unlockReg(truncated_reg_lock); + + try self.truncRegister( + dest_reg, + truncated_reg, + int_info.signedness, + int_info.bits, + ); + + _ = try self.addInst(.{ + .tag = .cmp, + .data = .{ .arithmetic_2op = .{ + .is_imm = false, + .rs1 = dest_reg, + .rs2_or_imm = .{ .rs2 = truncated_reg }, + } }, + }); + + const cond = Instruction.ICondition.ne; + const ccr = Instruction.CCR.xcc; + + break :result MCValue{ .register_with_overflow = .{ + .reg = truncated_reg, + .flag = .{ .cond = cond, .ccr = ccr }, + } }; + }, + // XXX DO NOT call __multi3 directly as it'll result in us doing six multiplications, + // which is far more than strictly necessary + 33...64 => return self.fail("TODO copy compiler-rt's mulddi3 for a 64x64->128 multiply", .{}), + else => return self.fail("TODO overflow operations on other integer sizes", .{}), + } + }, + else => unreachable, + } + }; + return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airNot(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const pt = self.pt; + const zcu = pt.zcu; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + switch (operand) { + .dead => unreachable, + .unreach => unreachable, + .condition_flags => |op| { + break :result MCValue{ + .condition_flags = .{ + .cond = op.cond.negate(), + .ccr = op.ccr, + }, + }; + }, + else => { + switch (operand_ty.zigTypeTag(zcu)) { + .bool => { + const op_reg = switch (operand) { + .register => |r| r, + else => try self.copyToTmpRegister(operand_ty, operand), + }; + const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg); + defer self.register_manager.unlockReg(reg_lock); + + const dest_reg = blk: { + if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { + break :blk op_reg; + } + + const reg = try self.register_manager.allocReg(null, gp); + break :blk reg; + }; + + _ = try self.addInst(.{ + .tag = .xor, + .data = .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = op_reg, + .rs2_or_imm = .{ .imm = 1 }, + }, + }, + }); + + break :result MCValue{ .register = dest_reg }; + }, + .vector => return self.fail("TODO bitwise not for vectors", .{}), + .int => { + const int_info = operand_ty.intInfo(zcu); + if (int_info.bits <= 64) { + const op_reg = switch (operand) { + .register => |r| r, + else => try self.copyToTmpRegister(operand_ty, operand), + }; + const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg); + defer self.register_manager.unlockReg(reg_lock); + + const dest_reg = blk: { + if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { + break :blk op_reg; + } + + const reg = try self.register_manager.allocReg(null, gp); + break :blk reg; + }; + + _ = try self.addInst(.{ + .tag = .not, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = dest_reg, + .rs2_or_imm = .{ .rs2 = op_reg }, + }, + }, + }); + + try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits); + + break :result MCValue{ .register = dest_reg }; + } else { + return self.fail("TODO sparc64 not on integers > u64/i64", .{}); + } + }, + else => unreachable, + } + }, + } + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void { + const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; + // TODO Emit a PREFETCH/IPREFETCH as necessary, see A.7 and A.42 + return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none }); +} + +fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { + const is_volatile = false; // TODO + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const ptr_bits = self.target.ptrBitWidth(); + const ptr_bytes = @divExact(ptr_bits, 8); + const mcv = try self.resolveInst(ty_op.operand); + switch (mcv) { + .dead, .unreach, .none => unreachable, + .ptr_stack_offset => |off| { + break :result MCValue{ .ptr_stack_offset = off - ptr_bytes }; + }, + else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}), + } + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const mcv = try self.resolveInst(ty_op.operand); + switch (mcv) { + .dead, .unreach, .none => unreachable, + .ptr_stack_offset => |off| { + break :result MCValue{ .ptr_stack_offset = off }; + }, + else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}), + } + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airRem(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + + // TODO add safety check + + // result = lhs - @divTrunc(lhs, rhs) * rhs + const result: MCValue = if (self.liveness.isUnused(inst)) blk: { + break :blk .dead; + } else blk: { + const tmp0 = try self.binOp(.div_trunc, lhs, rhs, lhs_ty, rhs_ty, null); + const tmp1 = try self.binOp(.mul, tmp0, rhs, lhs_ty, rhs_ty, null); + break :blk try self.binOp(.sub, lhs, tmp1, lhs_ty, rhs_ty, null); + }; + + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airRet(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + try self.ret(operand); + return self.finishAir(inst, .dead, .{ un_op, .none, .none }); +} + +fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ptr = try self.resolveInst(un_op); + _ = ptr; + return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch}); + //return self.finishAir(inst, .dead, .{ un_op, .none, .none }); +} + +fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { + const stack_offset = try self.allocMemPtr(inst); + return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none }); +} + +fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + _ = bin_op; + return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch}); +} + +fn airShlSat(self: *Self, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (self.liveness.isUnused(inst)) + .dead + else if (self.typeOf(bin_op.lhs).isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) + return self.fail("TODO implement vector shl_sat with scalar rhs for {}", .{self.target.cpu.arch}) + else + return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + const pt = self.pt; + const zcu = pt.zcu; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const lhs = try self.resolveInst(extra.lhs); + const rhs = try self.resolveInst(extra.rhs); + const lhs_ty = self.typeOf(extra.lhs); + const rhs_ty = self.typeOf(extra.rhs); + + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => if (!rhs_ty.isVector(zcu)) + return self.fail("TODO implement vector shl_with_overflow with scalar rhs", .{}) + else + return self.fail("TODO implement mul_with_overflow for vectors", .{}), + .int => { + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + try self.spillConditionFlagsIfOccupied(); + + const lhs_lock: ?RegisterLock = if (lhs == .register) + self.register_manager.lockRegAssumeUnused(lhs.register) + else + null; + // TODO this currently crashes stage1 + // defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); + + // Increase shift amount (i.e, rhs) by shamt_bits - int_info.bits + // e.g if shifting a i48 then use sr*x (shamt_bits == 64) but increase rhs by 16 + // and if shifting a i24 then use sr* (shamt_bits == 32) but increase rhs by 8 + const new_rhs = switch (int_info.bits) { + 1...31 => if (rhs == .immediate) MCValue{ + .immediate = rhs.immediate + 32 - int_info.bits, + } else try self.binOp(.add, rhs, .{ .immediate = 32 - int_info.bits }, rhs_ty, rhs_ty, null), + 33...63 => if (rhs == .immediate) MCValue{ + .immediate = rhs.immediate + 64 - int_info.bits, + } else try self.binOp(.add, rhs, .{ .immediate = 64 - int_info.bits }, rhs_ty, rhs_ty, null), + 32, 64 => rhs, + else => unreachable, + }; + + const new_rhs_lock: ?RegisterLock = if (new_rhs == .register) + self.register_manager.lockRegAssumeUnused(new_rhs.register) + else + null; + // TODO this currently crashes stage1 + // defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); + + const dest = try self.binOp(.shl, lhs, new_rhs, lhs_ty, rhs_ty, null); + const dest_reg = dest.register; + const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); + defer self.register_manager.unlockReg(dest_reg_lock); + + const shr = try self.binOp(.shr, dest, new_rhs, lhs_ty, rhs_ty, null); + + _ = try self.addInst(.{ + .tag = .cmp, + .data = .{ .arithmetic_2op = .{ + .is_imm = false, + .rs1 = dest_reg, + .rs2_or_imm = .{ .rs2 = shr.register }, + } }, + }); + + const cond = Instruction.ICondition.ne; + const ccr = switch (int_info.bits) { + 1...32 => Instruction.CCR.icc, + 33...64 => Instruction.CCR.xcc, + else => unreachable, + }; + + // TODO Those should really be written as defers, however stage1 currently + // panics when those are turned into defer statements so those are + // written here at the end as ordinary statements. + // Because of that, on failure, the lock on those registers wouldn't be + // released. + if (lhs_lock) |reg| self.register_manager.unlockReg(reg); + if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); + + break :result MCValue{ .register_with_overflow = .{ + .reg = dest_reg, + .flag = .{ .cond = cond, .ccr = ccr }, + } }; + } else { + return self.fail("TODO overflow operations on other integer sizes", .{}); + } + }, + else => unreachable, + } + }; + return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airSlice(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const ptr = try self.resolveInst(bin_op.lhs); + const ptr_ty = self.typeOf(bin_op.lhs); + const len = try self.resolveInst(bin_op.rhs); + const len_ty = self.typeOf(bin_op.rhs); + const ptr_bytes = 8; + const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8"); + try self.genSetStack(ptr_ty, stack_offset, ptr); + try self.genSetStack(len_ty, stack_offset - ptr_bytes, len); + break :result MCValue{ .stack_offset = stack_offset }; + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const is_volatile = false; // TODO + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); + const result: MCValue = result: { + const slice_mcv = try self.resolveInst(bin_op.lhs); + const index_mcv = try self.resolveInst(bin_op.rhs); + + const slice_ty = self.typeOf(bin_op.lhs); + const elem_ty = slice_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); + + const index_lock: ?RegisterLock = if (index_mcv == .register) + self.register_manager.lockRegAssumeUnused(index_mcv.register) + else + null; + defer if (index_lock) |reg| self.register_manager.unlockReg(reg); + + const base_mcv: MCValue = switch (slice_mcv) { + .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) }, + else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}), + }; + const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register); + defer self.register_manager.unlockReg(base_lock); + + switch (elem_size) { + else => { + // TODO skip the ptr_add emission entirely and use native addressing modes + // i.e sllx/mulx then R+R or scale immediate then R+I + const dest = try self.allocRegOrMem(inst, true); + const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ptr_field_type, Type.usize, null); + try self.load(dest, addr, slice_ptr_field_type); + + break :result dest; + }, + } + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const ptr_bits = self.target.ptrBitWidth(); + const ptr_bytes = @divExact(ptr_bits, 8); + const mcv = try self.resolveInst(ty_op.operand); + switch (mcv) { + .dead, .unreach, .none => unreachable, + .register => unreachable, // a slice doesn't fit in one register + .stack_offset => |off| { + break :result MCValue{ .stack_offset = off - ptr_bytes }; + }, + .memory => |addr| { + break :result MCValue{ .memory = addr + ptr_bytes }; + }, + else => return self.fail("TODO implement slice_len for {}", .{mcv}), + } + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const mcv = try self.resolveInst(ty_op.operand); + switch (mcv) { + .dead, .unreach, .none => unreachable, + .register => unreachable, // a slice doesn't fit in one register + .stack_offset => |off| { + break :result MCValue{ .stack_offset = off }; + }, + .memory => |addr| { + break :result MCValue{ .memory = addr }; + }, + else => return self.fail("TODO implement slice_len for {}", .{mcv}), + } + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSplat(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void { + if (safety) { + // TODO if the value is undef, write 0xaa bytes to dest + } else { + // TODO if the value is undef, don't lower this instruction + } + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ptr = try self.resolveInst(bin_op.lhs); + const value = try self.resolveInst(bin_op.rhs); + const ptr_ty = self.typeOf(bin_op.lhs); + const value_ty = self.typeOf(bin_op.rhs); + + try self.store(ptr, value, ptr_ty, value_ty); + + return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; + const result = try self.structFieldPtr(inst, extra.struct_operand, extra.field_index); + return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); +} + +fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = try self.structFieldPtr(inst, ty_op.operand, index); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; + const operand = extra.struct_operand; + const index = extra.field_index; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const zcu = self.pt.zcu; + const mcv = try self.resolveInst(operand); + const struct_ty = self.typeOf(operand); + const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu)); + + switch (mcv) { + .dead, .unreach => unreachable, + .stack_offset => |off| { + break :result MCValue{ .stack_offset = off - struct_field_offset }; + }, + .memory => |addr| { + break :result MCValue{ .memory = addr + struct_field_offset }; + }, + .register_with_overflow => |rwo| { + switch (index) { + 0 => { + // get wrapped value: return register + break :result MCValue{ .register = rwo.reg }; + }, + 1 => { + // TODO return special MCValue condition flags + // get overflow bit: set register to C flag + // resp. V flag + const dest_reg = try self.register_manager.allocReg(null, gp); + + // TODO handle floating point CCRs + assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc); + + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = dest_reg, + .rs2_or_imm = .{ .rs2 = .g0 }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .movcc, + .data = .{ + .conditional_move_int = .{ + .ccr = rwo.flag.ccr, + .cond = .{ .icond = rwo.flag.cond }, + .is_imm = true, + .rd = dest_reg, + .rs2_or_imm = .{ .imm = 1 }, + }, + }, + }); + + break :result MCValue{ .register = dest_reg }; + }, + else => unreachable, + } + }, + else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}), + } + }; + + return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); +} + +fn airSubSat(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { + _ = inst; + return self.fail("TODO implement switch for {}", .{self.target.cpu.arch}); +} + +fn airTagName(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else { + _ = operand; + return self.fail("TODO implement airTagName for {}", .{self.target.cpu.arch}); + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const dest_ty = self.typeOfIndex(inst); + + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: { + break :blk try self.trunc(inst, operand, operand_ty, dest_ty); + }; + + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airTry(self: *Self, inst: Air.Inst.Index) !void { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Try, pl_op.payload); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); + const result: MCValue = result: { + const error_union_ty = self.typeOf(pl_op.operand); + const error_union = try self.resolveInst(pl_op.operand); + const is_err_result = try self.isErr(error_union_ty, error_union); + const reloc = try self.condBr(is_err_result); + + try self.genBody(body); + + try self.performReloc(reloc); + break :result try self.errUnionPayload(error_union, error_union_ty); + }; + return self.finishAir(inst, result, .{ pl_op.operand, .none, .none }); +} + +fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const result: MCValue = if (self.liveness.isUnused(inst)) + .dead + else + return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; + _ = extra; + return self.fail("TODO implement airUnionInit for {}", .{self.target.cpu.arch}); +} + +fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const error_union_ty = self.typeOf(ty_op.operand); + const payload_ty = error_union_ty.errorUnionPayload(zcu); + const mcv = try self.resolveInst(ty_op.operand); + if (!payload_ty.hasRuntimeBits(zcu)) break :result mcv; + + return self.fail("TODO implement unwrap error union error for non-empty payloads", .{}); + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const error_union_ty = self.typeOf(ty_op.operand); + const payload_ty = error_union_ty.errorUnionPayload(zcu); + if (!payload_ty.hasRuntimeBits(zcu)) break :result MCValue.none; + + return self.fail("TODO implement unwrap error union payload for non-empty payloads", .{}); + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +/// E to E!T +fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const error_union_ty = ty_op.ty.toType(); + const payload_ty = error_union_ty.errorUnionPayload(zcu); + const mcv = try self.resolveInst(ty_op.operand); + if (!payload_ty.hasRuntimeBits(zcu)) break :result mcv; + + return self.fail("TODO implement wrap errunion error for non-empty payloads", .{}); + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +/// T to E!T +fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch}); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void { + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const optional_ty = self.typeOfIndex(inst); + + // Optional with a zero-bit payload type is just a boolean true + if (optional_ty.abiSize(pt.zcu) == 1) + break :result MCValue{ .immediate = 1 }; + + return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch}); + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +// Common helper functions + +fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { + const gpa = self.gpa; + try self.mir_instructions.ensureUnusedCapacity(gpa, 1); + const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len); + self.mir_instructions.appendAssumeCapacity(inst); + return result_index; +} + +fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: Alignment) !u32 { + self.stack_align = self.stack_align.max(abi_align); + // TODO find a free slot instead of always appending + const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset) + abi_size); + self.next_stack_offset = offset; + if (self.next_stack_offset > self.max_end_stack) + self.max_end_stack = self.next_stack_offset; + try self.stack.putNoClobber(self.gpa, offset, .{ + .inst = inst, + .size = abi_size, + }); + return offset; +} + +/// Use a pointer instruction as the basis for allocating stack memory. +fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 { + const pt = self.pt; + const zcu = pt.zcu; + const elem_ty = self.typeOfIndex(inst).childType(zcu); + + if (!elem_ty.hasRuntimeBits(zcu)) { + // As this stack item will never be dereferenced at runtime, + // return the stack offset 0. Stack offset 0 will be where all + // zero-sized stack allocations live as non-zero-sized + // allocations will always have an offset > 0. + return @as(u32, 0); + } + + const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse { + return self.fail("type '{f}' too big to fit into stack frame", .{elem_ty.fmt(pt)}); + }; + // TODO swap this for inst.ty.ptrAlign + const abi_align = elem_ty.abiAlignment(zcu); + return self.allocMem(inst, abi_size, abi_align); +} + +fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const elem_ty = self.typeOfIndex(inst); + const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse { + return self.fail("type '{f}' too big to fit into stack frame", .{elem_ty.fmt(pt)}); + }; + const abi_align = elem_ty.abiAlignment(zcu); + self.stack_align = self.stack_align.max(abi_align); + + if (reg_ok) { + // Make sure the type can fit in a register before we try to allocate one. + if (abi_size <= 8) { + if (self.register_manager.tryAllocReg(inst, gp)) |reg| { + return MCValue{ .register = reg }; + } + } + } + const stack_offset = try self.allocMem(inst, abi_size, abi_align); + return MCValue{ .stack_offset = stack_offset }; +} + +const BinOpMetadata = struct { + inst: Air.Inst.Index, + lhs: Air.Inst.Ref, + rhs: Air.Inst.Ref, +}; + +/// For all your binary operation needs, this function will generate +/// the corresponding Mir instruction(s). Returns the location of the +/// result. +/// +/// If the binary operation itself happens to be an Air instruction, +/// pass the corresponding index in the inst parameter. That helps +/// this function do stuff like reusing operands. +/// +/// This function does not do any lowering to Mir itself, but instead +/// looks at the lhs and rhs and determines which kind of lowering +/// would be best suitable and then delegates the lowering to other +/// functions. +fn binOp( + self: *Self, + tag: Air.Inst.Tag, + lhs: MCValue, + rhs: MCValue, + lhs_ty: Type, + rhs_ty: Type, + metadata: ?BinOpMetadata, +) InnerError!MCValue { + const pt = self.pt; + const zcu = pt.zcu; + switch (tag) { + .add, + .sub, + .mul, + .bit_and, + .bit_or, + .xor, + .cmp_eq, + => { + switch (lhs_ty.zigTypeTag(zcu)) { + .float => return self.fail("TODO binary operations on floats", .{}), + .vector => return self.fail("TODO binary operations on vectors", .{}), + .int => { + assert(lhs_ty.eql(rhs_ty, zcu)); + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + // Only say yes if the operation is + // commutative, i.e. we can swap both of the + // operands + const lhs_immediate_ok = switch (tag) { + .add => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .mul => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .bit_and => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .bit_or => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .xor => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .sub, .cmp_eq => false, + else => unreachable, + }; + const rhs_immediate_ok = switch (tag) { + .add, + .sub, + .mul, + .bit_and, + .bit_or, + .xor, + .cmp_eq, + => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), + else => unreachable, + }; + + const mir_tag: Mir.Inst.Tag = switch (tag) { + .add => .add, + .sub => .sub, + .mul => .mulx, + .bit_and => .@"and", + .bit_or => .@"or", + .xor => .xor, + .cmp_eq => .cmp, + else => unreachable, + }; + + if (rhs_immediate_ok) { + return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata); + } else if (lhs_immediate_ok) { + // swap lhs and rhs + return try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, metadata); + } else { + // TODO convert large immediates to register before adding + return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + } + } else { + return self.fail("TODO binary operations on int with bits > 64", .{}); + } + }, + else => unreachable, + } + }, + + .add_wrap, + .sub_wrap, + .mul_wrap, + => { + const base_tag: Air.Inst.Tag = switch (tag) { + .add_wrap => .add, + .sub_wrap => .sub, + .mul_wrap => .mul, + else => unreachable, + }; + + // Generate the base operation + const result = try self.binOp(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + + // Truncate if necessary + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO binary operations on vectors", .{}), + .int => { + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + const result_reg = result.register; + try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits); + return result; + } else { + return self.fail("TODO binary operations on integers > u64/i64", .{}); + } + }, + else => unreachable, + } + }, + + .div_trunc => { + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO binary operations on vectors", .{}), + .int => { + assert(lhs_ty.eql(rhs_ty, zcu)); + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + const rhs_immediate_ok = switch (tag) { + .div_trunc => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), + else => unreachable, + }; + + const mir_tag: Mir.Inst.Tag = switch (tag) { + .div_trunc => switch (int_info.signedness) { + .signed => Mir.Inst.Tag.sdivx, + .unsigned => Mir.Inst.Tag.udivx, + }, + else => unreachable, + }; + + if (rhs_immediate_ok) { + return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, true, metadata); + } else { + return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + } + } else { + return self.fail("TODO binary operations on int with bits > 64", .{}); + } + }, + else => unreachable, + } + }, + + .ptr_add => { + switch (lhs_ty.zigTypeTag(zcu)) { + .pointer => { + const ptr_ty = lhs_ty; + const elem_ty = switch (ptr_ty.ptrSize(zcu)) { + .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type + else => ptr_ty.childType(zcu), + }; + const elem_size = elem_ty.abiSize(zcu); + + if (elem_size == 1) { + const base_tag: Mir.Inst.Tag = switch (tag) { + .ptr_add => .add, + else => unreachable, + }; + + return try self.binOpRegister(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + } else { + // convert the offset into a byte offset by + // multiplying it with elem_size + + const offset = try self.binOp(.mul, rhs, .{ .immediate = elem_size }, Type.usize, Type.usize, null); + const addr = try self.binOp(tag, lhs, offset, Type.manyptr_u8, Type.usize, null); + return addr; + } + }, + else => unreachable, + } + }, + + .bool_and, + .bool_or, + => { + switch (lhs_ty.zigTypeTag(zcu)) { + .bool => { + assert(lhs != .immediate); // should have been handled by Sema + assert(rhs != .immediate); // should have been handled by Sema + + const mir_tag: Mir.Inst.Tag = switch (tag) { + .bool_and => .@"and", + .bool_or => .@"or", + else => unreachable, + }; + + return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + }, + else => unreachable, + } + }, + + .shl, + .shr, + => { + const base_tag: Air.Inst.Tag = switch (tag) { + .shl => .shl_exact, + .shr => .shr_exact, + else => unreachable, + }; + + // Generate the base operation + const result = try self.binOp(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + + // Truncate if necessary + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => if (rhs_ty.isVector(zcu)) + return self.fail("TODO vector shift with scalar rhs", .{}) + else + return self.fail("TODO binary operations on vectors", .{}), + .int => { + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + // 32 and 64 bit operands doesn't need truncating + if (int_info.bits == 32 or int_info.bits == 64) return result; + + const result_reg = result.register; + try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits); + return result; + } else { + return self.fail("TODO binary operations on integers > u64/i64", .{}); + } + }, + else => unreachable, + } + }, + + .shl_exact, + .shr_exact, + => { + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => if (rhs_ty.isVector(zcu)) + return self.fail("TODO vector shift with scalar rhs", .{}) + else + return self.fail("TODO binary operations on vectors", .{}), + .int => { + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + const rhs_immediate_ok = rhs == .immediate; + + const mir_tag: Mir.Inst.Tag = switch (tag) { + .shl_exact => if (int_info.bits <= 32) Mir.Inst.Tag.sll else Mir.Inst.Tag.sllx, + .shr_exact => switch (int_info.signedness) { + .signed => if (int_info.bits <= 32) Mir.Inst.Tag.sra else Mir.Inst.Tag.srax, + .unsigned => if (int_info.bits <= 32) Mir.Inst.Tag.srl else Mir.Inst.Tag.srlx, + }, + else => unreachable, + }; + + if (rhs_immediate_ok) { + return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata); + } else { + return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); + } + } else { + return self.fail("TODO binary operations on int with bits > 64", .{}); + } + }, + else => unreachable, + } + }, + + else => return self.fail("TODO implement {} binOp for SPARCv9", .{tag}), + } +} + +/// Don't call this function directly. Use binOp instead. +/// +/// Calling this function signals an intention to generate a Mir +/// instruction of the form +/// +/// op dest, lhs, #rhs_imm +/// +/// Set lhs_and_rhs_swapped to true iff inst.bin_op.lhs corresponds to +/// rhs and vice versa. This parameter is only used when metadata != null. +/// +/// Asserts that generating an instruction of that form is possible. +fn binOpImmediate( + self: *Self, + mir_tag: Mir.Inst.Tag, + lhs: MCValue, + rhs: MCValue, + lhs_ty: Type, + lhs_and_rhs_swapped: bool, + metadata: ?BinOpMetadata, +) !MCValue { + const lhs_is_register = lhs == .register; + + const lhs_lock: ?RegisterLock = if (lhs_is_register) + self.register_manager.lockReg(lhs.register) + else + null; + defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); + + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + + const lhs_reg = if (lhs_is_register) lhs.register else blk: { + const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: { + break :inst (if (lhs_and_rhs_swapped) md.rhs else md.lhs).toIndex().?; + } else null; + + const reg = try self.register_manager.allocReg(track_inst, gp); + + if (track_inst) |inst| { + const mcv: MCValue = .{ .register = reg }; + log.debug("binOpRegister move lhs %{d} to register: {} -> {}", .{ inst, lhs, mcv }); + branch.inst_table.putAssumeCapacity(inst, mcv); + + // If we're moving a condition flag MCV to register, + // mark it as free. + if (lhs == .condition_flags) { + assert(self.condition_flags_inst.? == inst); + self.condition_flags_inst = null; + } + } + + break :blk reg; + }; + const new_lhs_lock = self.register_manager.lockReg(lhs_reg); + defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); + + const dest_reg = switch (mir_tag) { + .cmp => undefined, // cmp has no destination register + else => if (metadata) |md| blk: { + if (lhs_is_register and self.reuseOperand( + md.inst, + if (lhs_and_rhs_swapped) md.rhs else md.lhs, + if (lhs_and_rhs_swapped) 1 else 0, + lhs, + )) { + break :blk lhs_reg; + } else { + break :blk try self.register_manager.allocReg(md.inst, gp); + } + } else blk: { + break :blk try self.register_manager.allocReg(null, gp); + }, + }; + + if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); + + const mir_data: Mir.Inst.Data = switch (mir_tag) { + .add, + .addcc, + .@"and", + .@"or", + .xor, + .xnor, + .mulx, + .sdivx, + .udivx, + .sub, + .subcc, + => .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .imm = @as(u12, @intCast(rhs.immediate)) }, + }, + }, + .sll, + .srl, + .sra, + => .{ + .shift = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .imm = @as(u5, @intCast(rhs.immediate)) }, + }, + }, + .sllx, + .srlx, + .srax, + => .{ + .shift = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .imm = @as(u6, @intCast(rhs.immediate)) }, + }, + }, + .cmp => .{ + .arithmetic_2op = .{ + .is_imm = true, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .imm = @as(u12, @intCast(rhs.immediate)) }, + }, + }, + else => unreachable, + }; + + _ = try self.addInst(.{ + .tag = mir_tag, + .data = mir_data, + }); + + return MCValue{ .register = dest_reg }; +} + +/// Don't call this function directly. Use binOp instead. +/// +/// Calling this function signals an intention to generate a Mir +/// instruction of the form +/// +/// op dest, lhs, rhs +/// +/// Asserts that generating an instruction of that form is possible. +fn binOpRegister( + self: *Self, + mir_tag: Mir.Inst.Tag, + lhs: MCValue, + rhs: MCValue, + lhs_ty: Type, + rhs_ty: Type, + metadata: ?BinOpMetadata, +) !MCValue { + const lhs_is_register = lhs == .register; + const rhs_is_register = rhs == .register; + + const lhs_lock: ?RegisterLock = if (lhs_is_register) + self.register_manager.lockReg(lhs.register) + else + null; + defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg); + + const rhs_lock: ?RegisterLock = if (rhs_is_register) + self.register_manager.lockReg(rhs.register) + else + null; + defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); + + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + + const lhs_reg = if (lhs_is_register) lhs.register else blk: { + const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: { + break :inst md.lhs.toIndex().?; + } else null; + + const reg = try self.register_manager.allocReg(track_inst, gp); + if (track_inst) |inst| { + const mcv: MCValue = .{ .register = reg }; + log.debug("binOpRegister move lhs %{d} to register: {} -> {}", .{ inst, lhs, mcv }); + branch.inst_table.putAssumeCapacity(inst, mcv); + + // If we're moving a condition flag MCV to register, + // mark it as free. + if (lhs == .condition_flags) { + assert(self.condition_flags_inst.? == inst); + self.condition_flags_inst = null; + } + } + + break :blk reg; + }; + const new_lhs_lock = self.register_manager.lockReg(lhs_reg); + defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); + + const rhs_reg = if (rhs_is_register) rhs.register else blk: { + const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: { + break :inst md.rhs.toIndex().?; + } else null; + + const reg = try self.register_manager.allocReg(track_inst, gp); + if (track_inst) |inst| { + const mcv: MCValue = .{ .register = reg }; + log.debug("binOpRegister move rhs %{d} to register: {} -> {}", .{ inst, rhs, mcv }); + branch.inst_table.putAssumeCapacity(inst, mcv); + + // If we're moving a condition flag MCV to register, + // mark it as free. + if (rhs == .condition_flags) { + assert(self.condition_flags_inst.? == inst); + self.condition_flags_inst = null; + } + } + + break :blk reg; + }; + const new_rhs_lock = self.register_manager.lockReg(rhs_reg); + defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); + + const dest_reg = switch (mir_tag) { + .cmp => undefined, // cmp has no destination register + else => if (metadata) |md| blk: { + if (lhs_is_register and self.reuseOperand(md.inst, md.lhs, 0, lhs)) { + break :blk lhs_reg; + } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { + break :blk rhs_reg; + } else { + break :blk try self.register_manager.allocReg(md.inst, gp); + } + } else blk: { + break :blk try self.register_manager.allocReg(null, gp); + }, + }; + + if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); + if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); + + const mir_data: Mir.Inst.Data = switch (mir_tag) { + .add, + .addcc, + .@"and", + .@"or", + .xor, + .xnor, + .mulx, + .sdivx, + .udivx, + .sub, + .subcc, + => .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = dest_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + .sll, + .srl, + .sra, + .sllx, + .srlx, + .srax, + => .{ + .shift = .{ + .is_imm = false, + .rd = dest_reg, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + .cmp => .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = lhs_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + else => unreachable, + }; + + _ = try self.addInst(.{ + .tag = mir_tag, + .data = mir_data, + }); + + return MCValue{ .register = dest_reg }; +} + +fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { + const block_data = self.blocks.getPtr(block).?; + + const zcu = self.pt.zcu; + if (self.typeOf(operand).hasRuntimeBits(zcu)) { + const operand_mcv = try self.resolveInst(operand); + const block_mcv = block_data.mcv; + if (block_mcv == .none) { + block_data.mcv = switch (operand_mcv) { + .none, .dead, .unreach => unreachable, + .register, .stack_offset, .memory => operand_mcv, + .immediate => blk: { + const new_mcv = try self.allocRegOrMem(block, true); + try self.setRegOrMem(self.typeOfIndex(block), new_mcv, operand_mcv); + break :blk new_mcv; + }, + else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), + }; + } else { + try self.setRegOrMem(self.typeOfIndex(block), block_mcv, operand_mcv); + } + } + return self.brVoid(block); +} + +fn brVoid(self: *Self, block: Air.Inst.Index) !void { + const block_data = self.blocks.getPtr(block).?; + + // Emit a jump with a relocation. It will be patched up after the block ends. + try block_data.relocs.ensureUnusedCapacity(self.gpa, 1); + + const br_index = try self.addInst(.{ + .tag = .bpcc, + .data = .{ + .branch_predict_int = .{ + .ccr = .xcc, + .cond = .al, + .inst = undefined, // Will be filled by performReloc + }, + }, + }); + + // TODO Find a way to fill this delay slot + _ = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); + + block_data.relocs.appendAssumeCapacity(br_index); +} + +fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index { + // Here we either emit a BPcc for branching on CCR content, + // or emit a BPr to branch on register content. + const reloc: Mir.Inst.Index = switch (condition) { + .condition_flags => |flags| try self.addInst(.{ + .tag = .bpcc, + .data = .{ + .branch_predict_int = .{ + .ccr = flags.ccr, + // Here we map to the opposite condition because the jump is to the false branch. + .cond = flags.cond.icond.negate(), + .inst = undefined, // Will be filled by performReloc + }, + }, + }), + .condition_register => |reg| try self.addInst(.{ + .tag = .bpr, + .data = .{ + .branch_predict_reg = .{ + .rs1 = reg.reg, + // Here we map to the opposite condition because the jump is to the false branch. + .cond = reg.cond.negate(), + .inst = undefined, // Will be filled by performReloc + }, + }, + }), + else => blk: { + const reg = switch (condition) { + .register => |r| r, + else => try self.copyToTmpRegister(Type.bool, condition), + }; + + break :blk try self.addInst(.{ + .tag = .bpr, + .data = .{ + .branch_predict_reg = .{ + .cond = .eq_zero, + .rs1 = reg, + .inst = undefined, // populated later through performReloc + }, + }, + }); + }, + }; + + // Regardless of the branch type that's emitted, we need to reserve + // a space for the delay slot. + // TODO Find a way to fill this delay slot + _ = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); + + return reloc; +} + +/// Copies a value to a register without tracking the register. The register is not considered +/// allocated. A second call to `copyToTmpRegister` may return the same register. +/// This can have a side effect of spilling instructions to the stack to free up a register. +fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { + const reg = try self.register_manager.allocReg(null, gp); + try self.genSetReg(ty, reg, mcv); + return reg; +} + +fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void { + const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table; + try table.ensureUnusedCapacity(self.gpa, additional_count); +} + +/// Given an error union, returns the payload +fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const err_ty = error_union_ty.errorUnionSet(zcu); + const payload_ty = error_union_ty.errorUnionPayload(zcu); + if (err_ty.errorSetIsEmpty(zcu)) { + return error_union_mcv; + } + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return MCValue.none; + } + + const payload_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); + switch (error_union_mcv) { + .register => return self.fail("TODO errUnionPayload for registers", .{}), + .stack_offset => |off| { + return MCValue{ .stack_offset = off - payload_offset }; + }, + .memory => |addr| { + return MCValue{ .memory = addr + payload_offset }; + }, + else => unreachable, // invalid MCValue for an error union + } +} + +fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = self.pt.zcu; + const func = zcu.funcInfo(self.func_index); + const msg = try ErrorMsg.create(zcu.gpa, self.src_loc, format, args); + return zcu.codegenFailMsg(func.owner_nav, msg); +} + +fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = self.pt.zcu; + const func = zcu.funcInfo(self.func_index); + return zcu.codegenFailMsg(func.owner_nav, msg); +} + +/// Called when there are no operands, and the instruction is always unreferenced. +fn finishAirBookkeeping(self: *Self) void { + if (std.debug.runtime_safety) { + self.air_bookkeeping += 1; + } +} + +fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void { + const tomb_bits = self.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (self.reused_operands.isSet(op_index)) continue; + self.processDeath(op.toIndexAllowNone() orelse continue); + } + if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) { + log.debug("%{d} => {}", .{ inst, result }); + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + branch.inst_table.putAssumeCapacityNoClobber(inst, result); + + switch (result) { + .register => |reg| { + // In some cases (such as bitcast), an operand + // may be the same MCValue as the result. If + // that operand died and was a register, it + // was freed by processDeath. We have to + // "re-allocate" the register. + if (self.register_manager.isRegFree(reg)) { + self.register_manager.getRegAssumeFree(reg, inst); + } + }, + else => {}, + } + } + self.finishAirBookkeeping(); +} + +fn genArgDbgInfo(self: Self, name: []const u8, ty: Type, mcv: MCValue) !void { + // TODO: Add a pseudo-instruction or something to defer this work until Emit. + // We aren't allowed to interact with linker state here. + if (true) return; + switch (self.debug_output) { + .dwarf => |dw| switch (mcv) { + .register => |reg| try dw.genLocalDebugInfo( + .local_arg, + name, + ty, + .{ .reg = reg.dwarfNum() }, + ), + else => {}, + }, + else => {}, + } +} + +// TODO replace this to call to extern memcpy +fn genInlineMemcpy( + self: *Self, + src: Register, + dst: Register, + len: Register, + tmp: Register, +) !void { + // Here we assume that len > 0. + // Also we do the copy from end -> start address to save a register. + + // sub len, 1, len + _ = try self.addInst(.{ + .tag = .sub, + .data = .{ .arithmetic_3op = .{ + .is_imm = true, + .rs1 = len, + .rs2_or_imm = .{ .imm = 1 }, + .rd = len, + } }, + }); + + // loop: + // ldub [src + len], tmp + _ = try self.addInst(.{ + .tag = .ldub, + .data = .{ .arithmetic_3op = .{ + .is_imm = false, + .rs1 = src, + .rs2_or_imm = .{ .rs2 = len }, + .rd = tmp, + } }, + }); + + // stb tmp, [dst + len] + _ = try self.addInst(.{ + .tag = .stb, + .data = .{ .arithmetic_3op = .{ + .is_imm = false, + .rs1 = dst, + .rs2_or_imm = .{ .rs2 = len }, + .rd = tmp, + } }, + }); + + // brnz len, loop + _ = try self.addInst(.{ + .tag = .bpr, + .data = .{ .branch_predict_reg = .{ + .cond = .ne_zero, + .rs1 = len, + .inst = @as(u32, @intCast(self.mir_instructions.len - 2)), + } }, + }); + + // Delay slot: + // sub len, 1, len + _ = try self.addInst(.{ + .tag = .sub, + .data = .{ .arithmetic_3op = .{ + .is_imm = true, + .rs1 = len, + .rs2_or_imm = .{ .imm = 1 }, + .rd = len, + } }, + }); + + // end: +} + +fn genLoad(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void { + assert(off_type == Register or off_type == i13); + + const is_imm = (off_type == i13); + + switch (abi_size) { + 1, 2, 4, 8 => { + const tag: Mir.Inst.Tag = switch (abi_size) { + 1 => .ldub, + 2 => .lduh, + 4 => .lduw, + 8 => .ldx, + else => unreachable, // unexpected abi size + }; + + _ = try self.addInst(.{ + .tag = tag, + .data = .{ + .arithmetic_3op = .{ + .is_imm = is_imm, + .rd = value_reg, + .rs1 = addr_reg, + .rs2_or_imm = if (is_imm) .{ .imm = off } else .{ .rs2 = off }, + }, + }, + }); + }, + 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), + else => unreachable, + } +} + +fn genLoadASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void { + switch (abi_size) { + 1, 2, 4, 8 => { + const tag: Mir.Inst.Tag = switch (abi_size) { + 1 => .lduba, + 2 => .lduha, + 4 => .lduwa, + 8 => .ldxa, + else => unreachable, // unexpected abi size + }; + + _ = try self.addInst(.{ + .tag = tag, + .data = .{ + .mem_asi = .{ + .rd = value_reg, + .rs1 = addr_reg, + .rs2 = off_reg, + .asi = asi, + }, + }, + }); + }, + 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), + else => unreachable, + } +} + +fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + switch (mcv) { + .dead => unreachable, + .unreach, .none => return, // Nothing to do. + .condition_flags => |op| { + const condition = op.cond; + const ccr = op.ccr; + + // TODO handle floating point CCRs + assert(ccr == .xcc or ccr == .icc); + + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = reg, + .rs2_or_imm = .{ .rs2 = .g0 }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .movcc, + .data = .{ + .conditional_move_int = .{ + .ccr = ccr, + .cond = condition, + .is_imm = true, + .rd = reg, + .rs2_or_imm = .{ .imm = 1 }, + }, + }, + }); + }, + .condition_register => |op| { + const condition = op.cond; + const register = op.reg; + + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = reg, + .rs2_or_imm = .{ .rs2 = .g0 }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .movr, + .data = .{ + .conditional_move_reg = .{ + .cond = condition, + .is_imm = true, + .rd = reg, + .rs1 = register, + .rs2_or_imm = .{ .imm = 1 }, + }, + }, + }); + }, + .undef => { + if (!self.wantSafety()) + return; // The already existing value will do just fine. + // Write the debug undefined value. + return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }); + }, + .ptr_stack_offset => |off| { + const real_offset = realStackOffset(off); + const simm13 = math.cast(i13, real_offset) orelse + return self.fail("TODO larger stack offsets: {}", .{real_offset}); + + _ = try self.addInst(.{ + .tag = .add, + .data = .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = reg, + .rs1 = .sp, + .rs2_or_imm = .{ .imm = simm13 }, + }, + }, + }); + }, + .immediate => |x| { + if (x <= math.maxInt(u12)) { + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = true, + .rs1 = reg, + .rs2_or_imm = .{ .imm = @as(u12, @truncate(x)) }, + }, + }, + }); + } else if (x <= math.maxInt(u32)) { + _ = try self.addInst(.{ + .tag = .sethi, + .data = .{ + .sethi = .{ + .rd = reg, + .imm = @as(u22, @truncate(x >> 10)), + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .@"or", + .data = .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = reg, + .rs1 = reg, + .rs2_or_imm = .{ .imm = @as(u10, @truncate(x)) }, + }, + }, + }); + } else if (x <= math.maxInt(u44)) { + try self.genSetReg(ty, reg, .{ .immediate = @as(u32, @truncate(x >> 12)) }); + + _ = try self.addInst(.{ + .tag = .sllx, + .data = .{ + .shift = .{ + .is_imm = true, + .rd = reg, + .rs1 = reg, + .rs2_or_imm = .{ .imm = 12 }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .@"or", + .data = .{ + .arithmetic_3op = .{ + .is_imm = true, + .rd = reg, + .rs1 = reg, + .rs2_or_imm = .{ .imm = @as(u12, @truncate(x)) }, + }, + }, + }); + } else { + // Need to allocate a temporary register to load 64-bit immediates. + const tmp_reg = try self.register_manager.allocReg(null, gp); + + try self.genSetReg(ty, tmp_reg, .{ .immediate = @as(u32, @truncate(x)) }); + try self.genSetReg(ty, reg, .{ .immediate = @as(u32, @truncate(x >> 32)) }); + + _ = try self.addInst(.{ + .tag = .sllx, + .data = .{ + .shift = .{ + .is_imm = true, + .rd = reg, + .rs1 = reg, + .rs2_or_imm = .{ .imm = 32 }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .@"or", + .data = .{ + .arithmetic_3op = .{ + .is_imm = false, + .rd = reg, + .rs1 = reg, + .rs2_or_imm = .{ .rs2 = tmp_reg }, + }, + }, + }); + } + }, + .register => |src_reg| { + // If the registers are the same, nothing to do. + if (src_reg.id() == reg.id()) + return; + + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = reg, + .rs2_or_imm = .{ .rs2 = src_reg }, + }, + }, + }); + }, + .register_with_overflow => unreachable, + .memory => |addr| { + // The value is in memory at a hard-coded address. + // If the type is a pointer, it means the pointer address is at this memory location. + try self.genSetReg(ty, reg, .{ .immediate = addr }); + try self.genLoad(reg, reg, i13, 0, ty.abiSize(zcu)); + }, + .stack_offset => |off| { + const real_offset = realStackOffset(off); + const simm13 = math.cast(i13, real_offset) orelse + return self.fail("TODO larger stack offsets: {}", .{real_offset}); + try self.genLoad(reg, .sp, i13, simm13, ty.abiSize(zcu)); + }, + } +} + +fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size = ty.abiSize(zcu); + switch (mcv) { + .dead => unreachable, + .unreach, .none => return, // Nothing to do. + .undef => { + if (!self.wantSafety()) + return; // The already existing value will do just fine. + // TODO Upgrade this to a memset call when we have that available. + switch (ty.abiSize(zcu)) { + 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }), + 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }), + 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }), + 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), + else => return self.fail("TODO implement memset", .{}), + } + }, + .condition_flags, + .condition_register, + .immediate, + .ptr_stack_offset, + => { + const reg = try self.copyToTmpRegister(ty, mcv); + return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }); + }, + .register => |reg| { + const real_offset = realStackOffset(stack_offset); + const simm13 = math.cast(i13, real_offset) orelse + return self.fail("TODO larger stack offsets: {}", .{real_offset}); + return self.genStore(reg, .sp, i13, simm13, abi_size); + }, + .register_with_overflow => |rwo| { + const reg_lock = self.register_manager.lockReg(rwo.reg); + defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg); + + const wrapped_ty = ty.fieldType(0, zcu); + try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg }); + + const overflow_bit_ty = ty.fieldType(1, zcu); + const overflow_bit_offset: u32 = @intCast(ty.structFieldOffset(1, zcu)); + const cond_reg = try self.register_manager.allocReg(null, gp); + + // TODO handle floating point CCRs + assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc); + + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = cond_reg, + .rs2_or_imm = .{ .rs2 = .g0 }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .movcc, + .data = .{ + .conditional_move_int = .{ + .ccr = rwo.flag.ccr, + .cond = .{ .icond = rwo.flag.cond }, + .is_imm = true, + .rd = cond_reg, + .rs2_or_imm = .{ .imm = 1 }, + }, + }, + }); + try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{ + .register = cond_reg, + }); + }, + .memory, .stack_offset => { + switch (mcv) { + .stack_offset => |off| { + if (stack_offset == off) + return; // Copy stack variable to itself; nothing to do. + }, + else => {}, + } + + if (abi_size <= 8) { + const reg = try self.copyToTmpRegister(ty, mcv); + return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }); + } else { + const ptr_ty = try pt.singleMutPtrType(ty); + + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp); + const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); + defer for (regs_locks) |reg| { + self.register_manager.unlockReg(reg); + }; + + const src_reg = regs[0]; + const dst_reg = regs[1]; + const len_reg = regs[2]; + const tmp_reg = regs[3]; + + switch (mcv) { + .stack_offset => |off| try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off }), + .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }), + else => unreachable, + } + + try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset }); + try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size }); + try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg); + } + }, + } +} + +fn genStore(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void { + assert(off_type == Register or off_type == i13); + + const is_imm = (off_type == i13); + + switch (abi_size) { + 1, 2, 4, 8 => { + const tag: Mir.Inst.Tag = switch (abi_size) { + 1 => .stb, + 2 => .sth, + 4 => .stw, + 8 => .stx, + else => unreachable, // unexpected abi size + }; + + _ = try self.addInst(.{ + .tag = tag, + .data = .{ + .arithmetic_3op = .{ + .is_imm = is_imm, + .rd = value_reg, + .rs1 = addr_reg, + .rs2_or_imm = if (is_imm) .{ .imm = off } else .{ .rs2 = off }, + }, + }, + }); + }, + 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), + else => unreachable, + } +} + +fn genStoreASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void { + switch (abi_size) { + 1, 2, 4, 8 => { + const tag: Mir.Inst.Tag = switch (abi_size) { + 1 => .stba, + 2 => .stha, + 4 => .stwa, + 8 => .stxa, + else => unreachable, // unexpected abi size + }; + + _ = try self.addInst(.{ + .tag = tag, + .data = .{ + .mem_asi = .{ + .rd = value_reg, + .rs1 = addr_reg, + .rs2 = off_reg, + .asi = asi, + }, + }, + }); + }, + 3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}), + else => unreachable, + } +} + +fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { + const pt = self.pt; + const mcv: MCValue = switch (try codegen.genTypedValue( + self.bin_file, + pt, + self.src_loc, + val, + self.target, + )) { + .mcv => |mcv| switch (mcv) { + .none => .none, + .undef => .undef, + .load_got, .load_symbol, .load_direct, .lea_symbol, .lea_direct => unreachable, // TODO + .immediate => |imm| .{ .immediate = imm }, + .memory => |addr| .{ .memory = addr }, + }, + .fail => |msg| { + self.err_msg = msg; + return error.CodegenFail; + }, + }; + return mcv; +} + +fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue { + // Treat each stack item as a "layer" on top of the previous one. + var i: usize = self.branch_stack.items.len; + while (true) { + i -= 1; + if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| { + log.debug("getResolvedInstValue %{f} => {}", .{ inst, mcv }); + assert(mcv != .dead); + return mcv; + } + } +} + +fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const error_type = ty.errorUnionSet(zcu); + const payload_type = ty.errorUnionPayload(zcu); + + if (!error_type.hasRuntimeBits(zcu)) { + return MCValue{ .immediate = 0 }; // always false + } else if (!payload_type.hasRuntimeBits(zcu)) { + if (error_type.abiSize(zcu) <= 8) { + const reg_mcv: MCValue = switch (operand) { + .register => operand, + else => .{ .register = try self.copyToTmpRegister(error_type, operand) }, + }; + + _ = try self.addInst(.{ + .tag = .cmp, + .data = .{ .arithmetic_2op = .{ + .is_imm = true, + .rs1 = reg_mcv.register, + .rs2_or_imm = .{ .imm = 0 }, + } }, + }); + + return MCValue{ .condition_flags = .{ .cond = .{ .icond = .gu }, .ccr = .xcc } }; + } else { + return self.fail("TODO isErr for errors with size > 8", .{}); + } + } else { + return self.fail("TODO isErr for non-empty payloads", .{}); + } +} + +fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue { + // Call isErr, then negate the result. + const is_err_result = try self.isErr(ty, operand); + switch (is_err_result) { + .condition_flags => |op| { + return MCValue{ .condition_flags = .{ .cond = op.cond.negate(), .ccr = op.ccr } }; + }, + .immediate => |imm| { + assert(imm == 0); + return MCValue{ .immediate = 1 }; + }, + else => unreachable, + } +} + +fn isNull(self: *Self, operand: MCValue) !MCValue { + _ = operand; + // Here you can specialize this instruction if it makes sense to, otherwise the default + // will call isNonNull and invert the result. + return self.fail("TODO call isNonNull and invert the result", .{}); +} + +fn isNonNull(self: *Self, operand: MCValue) !MCValue { + // Call isNull, then negate the result. + const is_null_result = try self.isNull(operand); + switch (is_null_result) { + .condition_flags => |op| { + return MCValue{ .condition_flags = .{ .cond = op.cond.negate(), .ccr = op.ccr } }; + }, + .immediate => |imm| { + assert(imm == 0); + return MCValue{ .immediate = 1 }; + }, + else => unreachable, + } +} + +fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb { + try self.ensureProcessDeathCapacity(operand_count + 1); + return BigTomb{ + .function = self, + .inst = inst, + .lbt = self.liveness.iterateBigTomb(inst), + }; +} + +/// Send control flow to `inst`. +fn jump(self: *Self, inst: Mir.Inst.Index) !void { + _ = try self.addInst(.{ + .tag = .bpcc, + .data = .{ + .branch_predict_int = .{ + .cond = .al, + .ccr = .xcc, + .inst = inst, + }, + }, + }); + + // TODO find out a way to fill this delay slot + _ = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); +} + +fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const elem_ty = ptr_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + switch (ptr) { + .none => unreachable, + .undef => unreachable, + .unreach => unreachable, + .dead => unreachable, + .condition_flags, + .condition_register, + .register_with_overflow, + => unreachable, // cannot hold an address + .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }), + .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }), + .register => |addr_reg| { + const addr_reg_lock = self.register_manager.lockReg(addr_reg); + defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg); + + switch (dst_mcv) { + .dead => unreachable, + .undef => unreachable, + .condition_flags => unreachable, + .register => |dst_reg| { + try self.genLoad(dst_reg, addr_reg, i13, 0, elem_size); + }, + .stack_offset => |off| { + if (elem_size <= 8) { + const tmp_reg = try self.register_manager.allocReg(null, gp); + const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_reg_lock); + + try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); + try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); + } else { + const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp); + const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs); + defer for (regs_locks) |reg| { + self.register_manager.unlockReg(reg); + }; + + const src_reg = addr_reg; + const dst_reg = regs[0]; + const len_reg = regs[1]; + const tmp_reg = regs[2]; + + try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off }); + try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size }); + try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg); + } + }, + else => return self.fail("TODO load from register into {}", .{dst_mcv}), + } + }, + .memory, + .stack_offset, + => { + const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr); + try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty); + }, + } +} + +fn minMax( + self: *Self, + tag: Air.Inst.Tag, + lhs: MCValue, + rhs: MCValue, + lhs_ty: Type, + rhs_ty: Type, +) InnerError!MCValue { + const pt = self.pt; + const zcu = pt.zcu; + assert(lhs_ty.eql(rhs_ty, zcu)); + switch (lhs_ty.zigTypeTag(zcu)) { + .float => return self.fail("TODO min/max on floats", .{}), + .vector => return self.fail("TODO min/max on vectors", .{}), + .int => { + const int_info = lhs_ty.intInfo(zcu); + if (int_info.bits <= 64) { + // TODO skip register setting when one of the operands + // is a small (fits in i13) immediate. + const rhs_is_register = rhs == .register; + const rhs_reg = if (rhs_is_register) + rhs.register + else + try self.register_manager.allocReg(null, gp); + const rhs_lock = self.register_manager.lockReg(rhs_reg); + defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); + if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); + + const result_reg = try self.register_manager.allocReg(null, gp); + const result_lock = self.register_manager.lockReg(result_reg); + defer if (result_lock) |reg| self.register_manager.unlockReg(reg); + try self.genSetReg(lhs_ty, result_reg, lhs); + + const cond_choose_rhs: Instruction.ICondition = switch (tag) { + .max => switch (int_info.signedness) { + .signed => Instruction.ICondition.gt, + .unsigned => Instruction.ICondition.gu, + }, + .min => switch (int_info.signedness) { + .signed => Instruction.ICondition.lt, + .unsigned => Instruction.ICondition.cs, + }, + else => unreachable, + }; + + _ = try self.addInst(.{ + .tag = .cmp, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = result_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + }); + + _ = try self.addInst(.{ + .tag = .movcc, + .data = .{ + .conditional_move_int = .{ + .is_imm = false, + .ccr = .xcc, + .cond = .{ .icond = cond_choose_rhs }, + .rd = result_reg, + .rs2_or_imm = .{ .rs2 = rhs_reg }, + }, + }, + }); + + return MCValue{ .register = result_reg }; + } else { + return self.fail("TODO min/max on integers > u64/i64", .{}); + } + }, + else => unreachable, + } +} + +fn parseRegName(name: []const u8) ?Register { + if (@hasDecl(Register, "parseRegName")) { + return Register.parseRegName(name); + } + return std.meta.stringToEnum(Register, name); +} + +fn performReloc(self: *Self, inst: Mir.Inst.Index) !void { + const tag = self.mir_instructions.items(.tag)[inst]; + switch (tag) { + .bpcc => self.mir_instructions.items(.data)[inst].branch_predict_int.inst = @intCast(self.mir_instructions.len), + .bpr => self.mir_instructions.items(.data)[inst].branch_predict_reg.inst = @intCast(self.mir_instructions.len), + else => unreachable, + } +} + +/// Asserts there is already capacity to insert into top branch inst_table. +fn processDeath(self: *Self, inst: Air.Inst.Index) void { + // When editing this function, note that the logic must synchronize with `reuseOperand`. + const prev_value = self.getResolvedInstValue(inst); + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + branch.inst_table.putAssumeCapacity(inst, .dead); + log.debug("%{f} death: {} -> .dead", .{ inst, prev_value }); + switch (prev_value) { + .register => |reg| { + self.register_manager.freeReg(reg); + }, + .register_with_overflow => |rwo| { + self.register_manager.freeReg(rwo.reg); + self.condition_flags_inst = null; + }, + .condition_flags => { + self.condition_flags_inst = null; + }, + else => {}, // TODO process stack allocation death + } +} + +/// Turns stack_offset MCV into a real SPARCv9 stack offset usable for asm. +fn realStackOffset(off: u32) u32 { + return off + + // SPARCv9 %sp points away from the stack by some amount. + abi.stack_bias + + // The first couple bytes of each stack frame is reserved + // for ABI and hardware purposes. + abi.stack_reserved_area; + // Only after that we have the usable stack frame portion. +} + +/// Caller must call `CallMCValues.deinit`. +fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: RegisterView) !CallMCValues { + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const fn_info = zcu.typeToFunc(fn_ty).?; + const cc = fn_info.cc; + var result: CallMCValues = .{ + .args = try self.gpa.alloc(MCValue, fn_info.param_types.len), + // These undefined values must be populated before returning from this function. + .return_value = undefined, + .stack_byte_count = undefined, + .stack_align = undefined, + }; + errdefer self.gpa.free(result.args); + + const ret_ty = fn_ty.fnReturnType(zcu); + + switch (cc) { + .naked => { + assert(result.args.len == 0); + result.return_value = .{ .unreach = {} }; + result.stack_byte_count = 0; + result.stack_align = .@"1"; + return result; + }, + .auto, .sparc64_sysv => { + // SPARC Compliance Definition 2.4.1, Chapter 3 + // Low-Level System Information (64-bit psABI) - Function Calling Sequence + + var next_register: usize = 0; + var next_stack_offset: u32 = 0; + // TODO: this is never assigned, which is a bug, but I don't know how this code works + // well enough to try and fix it. I *think* `next_register += next_stack_offset` is + // supposed to be `next_stack_offset += param_size` in every case where it appears. + _ = &next_stack_offset; + + // The caller puts the argument in %o0-%o5, which becomes %i0-%i5 inside the callee. + const argument_registers = switch (role) { + .caller => abi.c_abi_int_param_regs_caller_view, + .callee => abi.c_abi_int_param_regs_callee_view, + }; + + for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| { + const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu)); + if (param_size <= 8) { + if (next_register < argument_registers.len) { + result_arg.* = .{ .register = argument_registers[next_register] }; + next_register += 1; + } else { + result_arg.* = .{ .stack_offset = next_stack_offset }; + next_register += next_stack_offset; + } + } else if (param_size <= 16) { + if (next_register < argument_registers.len - 1) { + return self.fail("TODO MCValues with 2 registers", .{}); + } else if (next_register < argument_registers.len) { + return self.fail("TODO MCValues split register + stack", .{}); + } else { + result_arg.* = .{ .stack_offset = next_stack_offset }; + next_register += next_stack_offset; + } + } else { + result_arg.* = .{ .stack_offset = next_stack_offset }; + next_register += next_stack_offset; + } + } + + result.stack_byte_count = next_stack_offset; + result.stack_align = .@"16"; + + if (ret_ty.zigTypeTag(zcu) == .noreturn) { + result.return_value = .{ .unreach = {} }; + } else if (!ret_ty.hasRuntimeBits(zcu)) { + result.return_value = .{ .none = {} }; + } else { + const ret_ty_size: u32 = @intCast(ret_ty.abiSize(zcu)); + // The callee puts the return values in %i0-%i3, which becomes %o0-%o3 inside the caller. + if (ret_ty_size <= 8) { + result.return_value = switch (role) { + .caller => .{ .register = abi.c_abi_int_return_regs_caller_view[0] }, + .callee => .{ .register = abi.c_abi_int_return_regs_callee_view[0] }, + }; + } else { + return self.fail("TODO support more return values for sparc64", .{}); + } + } + }, + else => return self.fail("TODO implement function parameters for {} on sparc64", .{cc}), + } + + return result; +} + +fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue { + const pt = self.pt; + const ty = self.typeOf(ref); + + // If the type has no codegen bits, no need to store it. + if (!ty.hasRuntimeBitsIgnoreComptime(pt.zcu)) return .none; + + if (ref.toIndex()) |inst| { + return self.getResolvedInstValue(inst); + } + + return self.genTypedValue((try self.air.value(ref, pt)).?); +} + +fn ret(self: *Self, mcv: MCValue) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ret_ty = self.fn_type.fnReturnType(zcu); + try self.setRegOrMem(ret_ty, self.ret_mcv, mcv); + + // Just add space for a branch instruction, patch this later + const index = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); + + // Reserve space for the delay slot too + // TODO find out a way to fill this + _ = try self.addInst(.{ + .tag = .nop, + .data = .{ .nop = {} }, + }); + try self.exitlude_jump_relocs.append(self.gpa, index); +} + +fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Air.Liveness.OperandInt, mcv: MCValue) bool { + if (!self.liveness.operandDies(inst, op_index)) + return false; + + switch (mcv) { + .register => |reg| { + // If it's in the registers table, need to associate the register with the + // new instruction. + if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { + if (!self.register_manager.isRegFree(reg)) { + self.register_manager.registers[index] = inst; + } + } + log.debug("%{d} => {} (reused)", .{ inst, reg }); + }, + .stack_offset => |off| { + log.debug("%{d} => stack offset {d} (reused)", .{ inst, off }); + }, + else => return false, + } + + // Prevent the operand deaths processing code from deallocating it. + self.reused_operands.set(op_index); + + // That makes us responsible for doing the rest of the stuff that processDeath would have done. + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + branch.inst_table.putAssumeCapacity(operand.toIndex().?, .dead); + + return true; +} + +/// Sets the value without any modifications to register allocation metadata or stack allocation metadata. +fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void { + switch (loc) { + .none => return, + .register => |reg| return self.genSetReg(ty, reg, val), + .stack_offset => |off| return self.genSetStack(ty, off, val), + .memory => { + return self.fail("TODO implement setRegOrMem for memory", .{}); + }, + else => unreachable, + } +} + +/// Save the current instruction stored in the condition flags if +/// occupied +fn spillConditionFlagsIfOccupied(self: *Self) !void { + if (self.condition_flags_inst) |inst_to_save| { + const mcv = self.getResolvedInstValue(inst_to_save); + const new_mcv = switch (mcv) { + .condition_flags => try self.allocRegOrMem(inst_to_save, true), + .register_with_overflow => try self.allocRegOrMem(inst_to_save, false), + else => unreachable, // mcv doesn't occupy the compare flags + }; + + try self.setRegOrMem(self.typeOfIndex(inst_to_save), new_mcv, mcv); + log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv }); + + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + try branch.inst_table.put(self.gpa, inst_to_save, new_mcv); + + self.condition_flags_inst = null; + + // TODO consolidate with register manager and spillInstruction + // this call should really belong in the register manager! + switch (mcv) { + .register_with_overflow => |rwo| self.register_manager.freeReg(rwo.reg), + else => {}, + } + } +} + +pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { + const stack_mcv = try self.allocRegOrMem(inst, false); + log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv }); + const reg_mcv = self.getResolvedInstValue(inst); + assert(reg == reg_mcv.register); + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + try branch.inst_table.put(self.gpa, inst, stack_mcv); + try self.genSetStack(self.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv); +} + +fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { + const pt = self.pt; + const abi_size = value_ty.abiSize(pt.zcu); + + switch (ptr) { + .none => unreachable, + .undef => unreachable, + .unreach => unreachable, + .dead => unreachable, + .condition_flags, + .condition_register, + .register_with_overflow, + => unreachable, // cannot hold an address + .immediate => |imm| { + try self.setRegOrMem(value_ty, .{ .memory = imm }, value); + }, + .ptr_stack_offset => |off| { + try self.genSetStack(value_ty, off, value); + }, + .register => |addr_reg| { + const addr_reg_lock = self.register_manager.lockReg(addr_reg); + defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg); + + switch (value) { + .register => |value_reg| { + try self.genStore(value_reg, addr_reg, i13, 0, abi_size); + }, + else => { + return self.fail("TODO implement copying of memory", .{}); + }, + } + }, + .memory, + .stack_offset, + => { + const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr); + try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty); + }, + } +} + +fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue { + return if (self.liveness.isUnused(inst)) .dead else result: { + const pt = self.pt; + const zcu = pt.zcu; + const mcv = try self.resolveInst(operand); + const ptr_ty = self.typeOf(operand); + const struct_ty = ptr_ty.childType(zcu); + const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu)); + switch (mcv) { + .ptr_stack_offset => |off| { + break :result MCValue{ .ptr_stack_offset = off - struct_field_offset }; + }, + else => { + const offset_reg = try self.copyToTmpRegister(ptr_ty, .{ + .immediate = struct_field_offset, + }); + const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); + defer self.register_manager.unlockReg(offset_reg_lock); + + const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + + const dest = try self.binOp( + .add, + .{ .register = addr_reg }, + .{ .register = offset_reg }, + Type.usize, + Type.usize, + null, + ); + + break :result dest; + }, + } + }; +} + +fn trunc( + self: *Self, + maybe_inst: ?Air.Inst.Index, + operand: MCValue, + operand_ty: Type, + dest_ty: Type, +) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const info_a = operand_ty.intInfo(zcu); + const info_b = dest_ty.intInfo(zcu); + + if (info_b.bits <= 64) { + const operand_reg = switch (operand) { + .register => |r| r, + else => operand_reg: { + if (info_a.bits <= 64) { + const reg = try self.copyToTmpRegister(operand_ty, operand); + break :operand_reg reg; + } else { + return self.fail("TODO load least significant word into register", .{}); + } + }, + }; + const lock = self.register_manager.lockReg(operand_reg); + defer if (lock) |reg| self.register_manager.unlockReg(reg); + + const dest_reg = if (maybe_inst) |inst| blk: { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { + break :blk operand_reg; + } else { + const reg = try self.register_manager.allocReg(inst, gp); + break :blk reg; + } + } else blk: { + const reg = try self.register_manager.allocReg(null, gp); + break :blk reg; + }; + + try self.truncRegister(operand_reg, dest_reg, info_b.signedness, info_b.bits); + + return MCValue{ .register = dest_reg }; + } else { + return self.fail("TODO: truncate to ints > 64 bits", .{}); + } +} + +fn truncRegister( + self: *Self, + operand_reg: Register, + dest_reg: Register, + int_signedness: std.builtin.Signedness, + int_bits: u16, +) !void { + switch (int_bits) { + 1...31, 33...63 => { + _ = try self.addInst(.{ + .tag = .sllx, + .data = .{ + .shift = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = operand_reg, + .rs2_or_imm = .{ .imm = @as(u6, @intCast(64 - int_bits)) }, + }, + }, + }); + _ = try self.addInst(.{ + .tag = switch (int_signedness) { + .signed => .srax, + .unsigned => .srlx, + }, + .data = .{ + .shift = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = dest_reg, + .rs2_or_imm = .{ .imm = @as(u6, @intCast(int_bits)) }, + }, + }, + }); + }, + 32 => { + _ = try self.addInst(.{ + .tag = switch (int_signedness) { + .signed => .sra, + .unsigned => .srl, + }, + .data = .{ + .shift = .{ + .is_imm = true, + .rd = dest_reg, + .rs1 = operand_reg, + .rs2_or_imm = .{ .imm = 0 }, + }, + }, + }); + }, + 64 => { + if (dest_reg == operand_reg) + return; // Copy register to itself; nothing to do. + _ = try self.addInst(.{ + .tag = .mov, + .data = .{ + .arithmetic_2op = .{ + .is_imm = false, + .rs1 = dest_reg, + .rs2_or_imm = .{ .rs2 = operand_reg }, + }, + }, + }); + }, + else => unreachable, + } +} + +/// TODO support scope overrides. Also note this logic is duplicated with `Zcu.wantSafety`. +fn wantSafety(self: *Self) bool { + return switch (self.bin_file.comp.root_mod.optimize_mode) { + .Debug => true, + .ReleaseSafe => true, + .ReleaseFast => false, + .ReleaseSmall => false, + }; +} + +fn typeOf(self: *Self, inst: Air.Inst.Ref) Type { + return self.air.typeOf(inst, &self.pt.zcu.intern_pool); +} + +fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type { + return self.air.typeOfIndex(inst, &self.pt.zcu.intern_pool); +} diff --git a/src/codegen/sparc64/Emit.zig b/src/codegen/sparc64/Emit.zig new file mode 100644 index 0000000000000000000000000000000000000000..26282b09ab4f5c831cea32e9c9dbf8d6e0b2256a --- /dev/null +++ b/src/codegen/sparc64/Emit.zig @@ -0,0 +1,682 @@ +//! This file contains the functionality for lowering SPARCv9 MIR into +//! machine code + +const std = @import("std"); +const Endian = std.builtin.Endian; +const assert = std.debug.assert; +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); +const ErrorMsg = Zcu.ErrorMsg; +const log = std.log.scoped(.sparcv9_emit); + +const Emit = @This(); +const Mir = @import("Mir.zig"); +const bits = @import("bits.zig"); +const Instruction = bits.Instruction; +const Register = bits.Register; + +mir: Mir, +bin_file: *link.File, +debug_output: link.File.DebugInfoOutput, +target: *const std.Target, +err_msg: ?*ErrorMsg = null, +src_loc: Zcu.LazySrcLoc, +w: *std.Io.Writer, + +prev_di_line: u32, +prev_di_column: u32, +/// Relative to the beginning of `code`. +prev_di_pc: usize, + +/// The branch type of every branch +branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .empty, +/// For every forward branch, maps the target instruction to a list of +/// branches which branch to this target instruction +branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .empty, +/// For backward branches: stores the code offset of the target +/// instruction +/// +/// For forward branches: stores the code offset of the branch +/// instruction +code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty, + +const InnerError = std.Io.Writer.Error || error{ + OutOfMemory, + EmitFail, +}; + +const BranchType = enum { + bpcc, + bpr, + fn default(tag: Mir.Inst.Tag) BranchType { + return switch (tag) { + .bpcc => .bpcc, + .bpr => .bpr, + else => unreachable, + }; + } +}; + +pub fn emitMir( + emit: *Emit, +) InnerError!void { + const mir_tags = emit.mir.instructions.items(.tag); + + // Convert absolute addresses into offsets and + // find smallest lowerings for branch instructions + try emit.lowerBranches(); + + // Emit machine code + for (mir_tags, 0..) |tag, index| { + const inst = @as(u32, @intCast(index)); + switch (tag) { + .dbg_line => try emit.mirDbgLine(inst), + .dbg_prologue_end => try emit.mirDebugPrologueEnd(), + .dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(), + + .add => try emit.mirArithmetic3Op(inst), + .addcc => try emit.mirArithmetic3Op(inst), + + .bpr => try emit.mirConditionalBranch(inst), + .bpcc => try emit.mirConditionalBranch(inst), + + .call => @panic("TODO implement sparc64 call"), + + .jmpl => try emit.mirArithmetic3Op(inst), + + .ldub => try emit.mirArithmetic3Op(inst), + .lduh => try emit.mirArithmetic3Op(inst), + .lduw => try emit.mirArithmetic3Op(inst), + .ldx => try emit.mirArithmetic3Op(inst), + + .lduba => try emit.mirMemASI(inst), + .lduha => try emit.mirMemASI(inst), + .lduwa => try emit.mirMemASI(inst), + .ldxa => try emit.mirMemASI(inst), + + .@"and" => try emit.mirArithmetic3Op(inst), + .@"or" => try emit.mirArithmetic3Op(inst), + .xor => try emit.mirArithmetic3Op(inst), + .xnor => try emit.mirArithmetic3Op(inst), + + .membar => try emit.mirMembar(inst), + + .movcc => try emit.mirConditionalMove(inst), + + .movr => try emit.mirConditionalMove(inst), + + .mulx => try emit.mirArithmetic3Op(inst), + .sdivx => try emit.mirArithmetic3Op(inst), + .udivx => try emit.mirArithmetic3Op(inst), + + .nop => try emit.mirNop(), + + .@"return" => try emit.mirArithmetic2Op(inst), + + .save => try emit.mirArithmetic3Op(inst), + .restore => try emit.mirArithmetic3Op(inst), + + .sethi => try emit.mirSethi(inst), + + .sll => try emit.mirShift(inst), + .srl => try emit.mirShift(inst), + .sra => try emit.mirShift(inst), + .sllx => try emit.mirShift(inst), + .srlx => try emit.mirShift(inst), + .srax => try emit.mirShift(inst), + + .stb => try emit.mirArithmetic3Op(inst), + .sth => try emit.mirArithmetic3Op(inst), + .stw => try emit.mirArithmetic3Op(inst), + .stx => try emit.mirArithmetic3Op(inst), + + .stba => try emit.mirMemASI(inst), + .stha => try emit.mirMemASI(inst), + .stwa => try emit.mirMemASI(inst), + .stxa => try emit.mirMemASI(inst), + + .sub => try emit.mirArithmetic3Op(inst), + .subcc => try emit.mirArithmetic3Op(inst), + + .tcc => try emit.mirTrap(inst), + + .cmp => try emit.mirArithmetic2Op(inst), + + .mov => try emit.mirArithmetic2Op(inst), + + .not => try emit.mirArithmetic2Op(inst), + } + } +} + +pub fn deinit(emit: *Emit) void { + const comp = emit.bin_file.comp; + const gpa = comp.gpa; + var iter = emit.branch_forward_origins.valueIterator(); + while (iter.next()) |origin_list| { + origin_list.deinit(gpa); + } + + emit.branch_types.deinit(gpa); + emit.branch_forward_origins.deinit(gpa); + emit.code_offset_mapping.deinit(gpa); + emit.* = undefined; +} + +fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column; + + switch (tag) { + .dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column), + else => unreachable, + } +} + +fn mirDebugPrologueEnd(emit: *Emit) !void { + switch (emit.debug_output) { + .dwarf => |dbg_out| { + try dbg_out.setPrologueEnd(); + try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); + }, + .none => {}, + } +} + +fn mirDebugEpilogueBegin(emit: *Emit) !void { + switch (emit.debug_output) { + .dwarf => |dbg_out| { + try dbg_out.setEpilogueBegin(); + try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column); + }, + .none => {}, + } +} + +fn mirArithmetic2Op(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const data = emit.mir.instructions.items(.data)[inst].arithmetic_2op; + + const rs1 = data.rs1; + + if (data.is_imm) { + const imm = data.rs2_or_imm.imm; + switch (tag) { + .@"return" => try emit.writeInstruction(Instruction.@"return"(i13, rs1, imm)), + .cmp => try emit.writeInstruction(Instruction.subcc(i13, rs1, imm, .g0)), + .mov => try emit.writeInstruction(Instruction.@"or"(i13, .g0, imm, rs1)), + .not => try emit.writeInstruction(Instruction.xnor(i13, .g0, imm, rs1)), + else => unreachable, + } + } else { + const rs2 = data.rs2_or_imm.rs2; + switch (tag) { + .@"return" => try emit.writeInstruction(Instruction.@"return"(Register, rs1, rs2)), + .cmp => try emit.writeInstruction(Instruction.subcc(Register, rs1, rs2, .g0)), + .mov => try emit.writeInstruction(Instruction.@"or"(Register, .g0, rs2, rs1)), + .not => try emit.writeInstruction(Instruction.xnor(Register, rs2, .g0, rs1)), + else => unreachable, + } + } +} + +fn mirArithmetic3Op(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const data = emit.mir.instructions.items(.data)[inst].arithmetic_3op; + + const rd = data.rd; + const rs1 = data.rs1; + + if (data.is_imm) { + const imm = data.rs2_or_imm.imm; + switch (tag) { + .add => try emit.writeInstruction(Instruction.add(i13, rs1, imm, rd)), + .addcc => try emit.writeInstruction(Instruction.addcc(i13, rs1, imm, rd)), + .jmpl => try emit.writeInstruction(Instruction.jmpl(i13, rs1, imm, rd)), + .ldub => try emit.writeInstruction(Instruction.ldub(i13, rs1, imm, rd)), + .lduh => try emit.writeInstruction(Instruction.lduh(i13, rs1, imm, rd)), + .lduw => try emit.writeInstruction(Instruction.lduw(i13, rs1, imm, rd)), + .ldx => try emit.writeInstruction(Instruction.ldx(i13, rs1, imm, rd)), + .@"and" => try emit.writeInstruction(Instruction.@"and"(i13, rs1, imm, rd)), + .@"or" => try emit.writeInstruction(Instruction.@"or"(i13, rs1, imm, rd)), + .xor => try emit.writeInstruction(Instruction.xor(i13, rs1, imm, rd)), + .xnor => try emit.writeInstruction(Instruction.xnor(i13, rs1, imm, rd)), + .mulx => try emit.writeInstruction(Instruction.mulx(i13, rs1, imm, rd)), + .sdivx => try emit.writeInstruction(Instruction.sdivx(i13, rs1, imm, rd)), + .udivx => try emit.writeInstruction(Instruction.udivx(i13, rs1, imm, rd)), + .save => try emit.writeInstruction(Instruction.save(i13, rs1, imm, rd)), + .restore => try emit.writeInstruction(Instruction.restore(i13, rs1, imm, rd)), + .stb => try emit.writeInstruction(Instruction.stb(i13, rs1, imm, rd)), + .sth => try emit.writeInstruction(Instruction.sth(i13, rs1, imm, rd)), + .stw => try emit.writeInstruction(Instruction.stw(i13, rs1, imm, rd)), + .stx => try emit.writeInstruction(Instruction.stx(i13, rs1, imm, rd)), + .sub => try emit.writeInstruction(Instruction.sub(i13, rs1, imm, rd)), + .subcc => try emit.writeInstruction(Instruction.subcc(i13, rs1, imm, rd)), + else => unreachable, + } + } else { + const rs2 = data.rs2_or_imm.rs2; + switch (tag) { + .add => try emit.writeInstruction(Instruction.add(Register, rs1, rs2, rd)), + .addcc => try emit.writeInstruction(Instruction.addcc(Register, rs1, rs2, rd)), + .jmpl => try emit.writeInstruction(Instruction.jmpl(Register, rs1, rs2, rd)), + .ldub => try emit.writeInstruction(Instruction.ldub(Register, rs1, rs2, rd)), + .lduh => try emit.writeInstruction(Instruction.lduh(Register, rs1, rs2, rd)), + .lduw => try emit.writeInstruction(Instruction.lduw(Register, rs1, rs2, rd)), + .ldx => try emit.writeInstruction(Instruction.ldx(Register, rs1, rs2, rd)), + .@"and" => try emit.writeInstruction(Instruction.@"and"(Register, rs1, rs2, rd)), + .@"or" => try emit.writeInstruction(Instruction.@"or"(Register, rs1, rs2, rd)), + .xor => try emit.writeInstruction(Instruction.xor(Register, rs1, rs2, rd)), + .xnor => try emit.writeInstruction(Instruction.xnor(Register, rs1, rs2, rd)), + .mulx => try emit.writeInstruction(Instruction.mulx(Register, rs1, rs2, rd)), + .sdivx => try emit.writeInstruction(Instruction.sdivx(Register, rs1, rs2, rd)), + .udivx => try emit.writeInstruction(Instruction.udivx(Register, rs1, rs2, rd)), + .save => try emit.writeInstruction(Instruction.save(Register, rs1, rs2, rd)), + .restore => try emit.writeInstruction(Instruction.restore(Register, rs1, rs2, rd)), + .stb => try emit.writeInstruction(Instruction.stb(Register, rs1, rs2, rd)), + .sth => try emit.writeInstruction(Instruction.sth(Register, rs1, rs2, rd)), + .stw => try emit.writeInstruction(Instruction.stw(Register, rs1, rs2, rd)), + .stx => try emit.writeInstruction(Instruction.stx(Register, rs1, rs2, rd)), + .sub => try emit.writeInstruction(Instruction.sub(Register, rs1, rs2, rd)), + .subcc => try emit.writeInstruction(Instruction.subcc(Register, rs1, rs2, rd)), + else => unreachable, + } + } +} + +fn mirConditionalBranch(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const branch_type = emit.branch_types.get(inst).?; + + switch (branch_type) { + .bpcc => switch (tag) { + .bpcc => { + const branch_predict_int = emit.mir.instructions.items(.data)[inst].branch_predict_int; + const offset = @as(i64, @intCast(emit.code_offset_mapping.get(branch_predict_int.inst).?)) - @as(i64, @intCast(emit.w.end)); + log.debug("mirConditionalBranch: {} offset={}", .{ inst, offset }); + + try emit.writeInstruction( + Instruction.bpcc( + branch_predict_int.cond, + branch_predict_int.annul, + branch_predict_int.pt, + branch_predict_int.ccr, + @as(i21, @intCast(offset)), + ), + ); + }, + else => unreachable, + }, + .bpr => switch (tag) { + .bpr => { + const branch_predict_reg = emit.mir.instructions.items(.data)[inst].branch_predict_reg; + const offset = @as(i64, @intCast(emit.code_offset_mapping.get(branch_predict_reg.inst).?)) - @as(i64, @intCast(emit.w.end)); + log.debug("mirConditionalBranch: {} offset={}", .{ inst, offset }); + + try emit.writeInstruction( + Instruction.bpr( + branch_predict_reg.cond, + branch_predict_reg.annul, + branch_predict_reg.pt, + branch_predict_reg.rs1, + @as(i18, @intCast(offset)), + ), + ); + }, + else => unreachable, + }, + } +} + +fn mirConditionalMove(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + + switch (tag) { + .movcc => { + const data = emit.mir.instructions.items(.data)[inst].conditional_move_int; + if (data.is_imm) { + try emit.writeInstruction(Instruction.movcc( + i11, + data.cond, + data.ccr, + data.rs2_or_imm.imm, + data.rd, + )); + } else { + try emit.writeInstruction(Instruction.movcc( + Register, + data.cond, + data.ccr, + data.rs2_or_imm.rs2, + data.rd, + )); + } + }, + .movr => { + const data = emit.mir.instructions.items(.data)[inst].conditional_move_reg; + if (data.is_imm) { + try emit.writeInstruction(Instruction.movr( + i10, + data.cond, + data.rs1, + data.rs2_or_imm.imm, + data.rd, + )); + } else { + try emit.writeInstruction(Instruction.movr( + Register, + data.cond, + data.rs1, + data.rs2_or_imm.rs2, + data.rd, + )); + } + }, + else => unreachable, + } +} + +fn mirMemASI(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const data = emit.mir.instructions.items(.data)[inst].mem_asi; + + const rd = data.rd; + const rs1 = data.rs1; + const rs2 = data.rs2; + const asi = data.asi; + + switch (tag) { + .lduba => try emit.writeInstruction(Instruction.lduba(rs1, rs2, asi, rd)), + .lduha => try emit.writeInstruction(Instruction.lduha(rs1, rs2, asi, rd)), + .lduwa => try emit.writeInstruction(Instruction.lduwa(rs1, rs2, asi, rd)), + .ldxa => try emit.writeInstruction(Instruction.ldxa(rs1, rs2, asi, rd)), + + .stba => try emit.writeInstruction(Instruction.stba(rs1, rs2, asi, rd)), + .stha => try emit.writeInstruction(Instruction.stha(rs1, rs2, asi, rd)), + .stwa => try emit.writeInstruction(Instruction.stwa(rs1, rs2, asi, rd)), + .stxa => try emit.writeInstruction(Instruction.stxa(rs1, rs2, asi, rd)), + else => unreachable, + } +} + +fn mirMembar(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const mask = emit.mir.instructions.items(.data)[inst].membar_mask; + assert(tag == .membar); + + try emit.writeInstruction(Instruction.membar( + mask.cmask, + mask.mmask, + )); +} + +fn mirNop(emit: *Emit) !void { + try emit.writeInstruction(Instruction.nop()); +} + +fn mirSethi(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const data = emit.mir.instructions.items(.data)[inst].sethi; + + const imm = data.imm; + const rd = data.rd; + + assert(tag == .sethi); + try emit.writeInstruction(Instruction.sethi(imm, rd)); +} + +fn mirShift(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const data = emit.mir.instructions.items(.data)[inst].shift; + + const rd = data.rd; + const rs1 = data.rs1; + + if (data.is_imm) { + const imm = data.rs2_or_imm.imm; + switch (tag) { + .sll => try emit.writeInstruction(Instruction.sll(u5, rs1, @as(u5, @truncate(imm)), rd)), + .srl => try emit.writeInstruction(Instruction.srl(u5, rs1, @as(u5, @truncate(imm)), rd)), + .sra => try emit.writeInstruction(Instruction.sra(u5, rs1, @as(u5, @truncate(imm)), rd)), + .sllx => try emit.writeInstruction(Instruction.sllx(u6, rs1, imm, rd)), + .srlx => try emit.writeInstruction(Instruction.srlx(u6, rs1, imm, rd)), + .srax => try emit.writeInstruction(Instruction.srax(u6, rs1, imm, rd)), + else => unreachable, + } + } else { + const rs2 = data.rs2_or_imm.rs2; + switch (tag) { + .sll => try emit.writeInstruction(Instruction.sll(Register, rs1, rs2, rd)), + .srl => try emit.writeInstruction(Instruction.srl(Register, rs1, rs2, rd)), + .sra => try emit.writeInstruction(Instruction.sra(Register, rs1, rs2, rd)), + .sllx => try emit.writeInstruction(Instruction.sllx(Register, rs1, rs2, rd)), + .srlx => try emit.writeInstruction(Instruction.srlx(Register, rs1, rs2, rd)), + .srax => try emit.writeInstruction(Instruction.srax(Register, rs1, rs2, rd)), + else => unreachable, + } + } +} + +fn mirTrap(emit: *Emit, inst: Mir.Inst.Index) !void { + const tag = emit.mir.instructions.items(.tag)[inst]; + const data = emit.mir.instructions.items(.data)[inst].trap; + + const cond = data.cond; + const ccr = data.ccr; + const rs1 = data.rs1; + + if (data.is_imm) { + const imm = data.rs2_or_imm.imm; + switch (tag) { + .tcc => try emit.writeInstruction(Instruction.trap(u7, cond, ccr, rs1, imm)), + else => unreachable, + } + } else { + const rs2 = data.rs2_or_imm.rs2; + switch (tag) { + .tcc => try emit.writeInstruction(Instruction.trap(Register, cond, ccr, rs1, rs2)), + else => unreachable, + } + } +} + +// Common helper functions + +fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index { + const tag = emit.mir.instructions.items(.tag)[inst]; + + switch (tag) { + .bpcc => return emit.mir.instructions.items(.data)[inst].branch_predict_int.inst, + .bpr => return emit.mir.instructions.items(.data)[inst].branch_predict_reg.inst, + else => unreachable, + } +} + +fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) !void { + const delta_line = @as(i32, @intCast(line)) - @as(i32, @intCast(emit.prev_di_line)); + const delta_pc: usize = emit.w.end - emit.prev_di_pc; + switch (emit.debug_output) { + .dwarf => |dbg_out| { + try dbg_out.advancePCAndLine(delta_line, delta_pc); + emit.prev_di_line = line; + emit.prev_di_column = column; + emit.prev_di_pc = emit.w.end; + }, + else => {}, + } +} + +fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { + @branchHint(.cold); + assert(emit.err_msg == null); + const comp = emit.bin_file.comp; + const gpa = comp.gpa; + emit.err_msg = try ErrorMsg.create(gpa, emit.src_loc, format, args); + return error.EmitFail; +} + +fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize { + const tag = emit.mir.instructions.items(.tag)[inst]; + + switch (tag) { + .dbg_line, + .dbg_epilogue_begin, + .dbg_prologue_end, + => return 0, + // Currently Mir instructions always map to single machine instruction. + else => return 4, + } +} + +fn isBranch(tag: Mir.Inst.Tag) bool { + return switch (tag) { + .bpcc => true, + .bpr => true, + else => false, + }; +} + +fn lowerBranches(emit: *Emit) !void { + const comp = emit.bin_file.comp; + const gpa = comp.gpa; + const mir_tags = emit.mir.instructions.items(.tag); + + // First pass: Note down all branches and their target + // instructions, i.e. populate branch_types, + // branch_forward_origins, and code_offset_mapping + // + // TODO optimization opportunity: do this in codegen while + // generating MIR + for (mir_tags, 0..) |tag, index| { + const inst = @as(u32, @intCast(index)); + if (isBranch(tag)) { + const target_inst = emit.branchTarget(inst); + + // Remember this branch instruction + try emit.branch_types.put(gpa, inst, BranchType.default(tag)); + + // Forward branches require some extra stuff: We only + // know their offset once we arrive at the target + // instruction. Therefore, we need to be able to + // access the branch instruction when we visit the + // target instruction in order to manipulate its type + // etc. + if (target_inst > inst) { + // Remember the branch instruction index + try emit.code_offset_mapping.put(gpa, inst, 0); + + if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| { + try origin_list.append(gpa, inst); + } else { + var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty; + try origin_list.append(gpa, inst); + try emit.branch_forward_origins.put(gpa, target_inst, origin_list); + } + } + + // Remember the target instruction index so that we + // update the real code offset in all future passes + // + // putNoClobber may not be used as the put operation + // may clobber the entry when multiple branches branch + // to the same target instruction + try emit.code_offset_mapping.put(gpa, target_inst, 0); + } + } + + // Further passes: Until all branches are lowered, interate + // through all instructions and calculate new offsets and + // potentially new branch types + var all_branches_lowered = false; + while (!all_branches_lowered) { + all_branches_lowered = true; + var current_code_offset: usize = 0; + + for (mir_tags, 0..) |tag, index| { + const inst = @as(u32, @intCast(index)); + + // If this instruction contained in the code offset + // mapping (when it is a target of a branch or if it is a + // forward branch), update the code offset + if (emit.code_offset_mapping.getPtr(inst)) |offset| { + offset.* = current_code_offset; + } + + // If this instruction is a backward branch, calculate the + // offset, which may potentially update the branch type + if (isBranch(tag)) { + const target_inst = emit.branchTarget(inst); + if (target_inst < inst) { + const target_offset = emit.code_offset_mapping.get(target_inst).?; + const offset = @as(i64, @intCast(target_offset)) - @as(i64, @intCast(current_code_offset)); + const branch_type = emit.branch_types.getPtr(inst).?; + const optimal_branch_type = try emit.optimalBranchType(tag, offset); + if (branch_type.* != optimal_branch_type) { + branch_type.* = optimal_branch_type; + all_branches_lowered = false; + } + + log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset }); + } + } + + // If this instruction is the target of one or more + // forward branches, calculate the offset, which may + // potentially update the branch type + if (emit.branch_forward_origins.get(inst)) |origin_list| { + for (origin_list.items) |forward_branch_inst| { + const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst]; + const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?; + const offset = @as(i64, @intCast(current_code_offset)) - @as(i64, @intCast(forward_branch_inst_offset)); + const branch_type = emit.branch_types.getPtr(forward_branch_inst).?; + const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset); + if (branch_type.* != optimal_branch_type) { + branch_type.* = optimal_branch_type; + all_branches_lowered = false; + } + + log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset }); + } + } + + // Increment code offset + current_code_offset += emit.instructionSize(inst); + } + } +} + +fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType { + assert(offset & 0b11 == 0); + + switch (tag) { + // TODO use the following strategy to implement long branches: + // - Negate the conditional and target of the original instruction; + // - In the space immediately after the branch, load + // the address of the original target, preferrably in + // a PC-relative way, into %o7; and + // - jmpl %o7 + %g0, %g0 + + .bpcc => { + if (std.math.cast(i21, offset)) |_| { + return BranchType.bpcc; + } else { + return emit.fail("TODO support BPcc branches larger than +-1 MiB", .{}); + } + }, + .bpr => { + if (std.math.cast(i18, offset)) |_| { + return BranchType.bpr; + } else { + return emit.fail("TODO support BPr branches larger than +-128 KiB", .{}); + } + }, + else => unreachable, + } +} + +fn writeInstruction(emit: *Emit, instruction: Instruction) !void { + // SPARCv9 instructions are always arranged in BE regardless of the + // endianness mode the CPU is running in (Section 3.1 of the ISA specification). + // This is to ease porting in case someone wants to do a LE SPARCv9 backend. + try emit.w.writeInt(u32, instruction.toU32(), .big); +} diff --git a/src/codegen/sparc64/Mir.zig b/src/codegen/sparc64/Mir.zig new file mode 100644 index 0000000000000000000000000000000000000000..41d135eb831fefe51afd3fde7925596fce9d441b --- /dev/null +++ b/src/codegen/sparc64/Mir.zig @@ -0,0 +1,428 @@ +//! Machine Intermediate Representation. +//! This data is produced by SPARCv9 Codegen or SPARCv9 assembly parsing +//! These instructions have a 1:1 correspondence with machine code instructions +//! for the target. MIR can be lowered to source-annotated textual assembly code +//! instructions, or it can be lowered to machine code. +//! The main purpose of MIR is to postpone the assignment of offsets until Isel, +//! so that, for example, the smaller encodings of jump instructions can be used. + +const std = @import("std"); +const builtin = @import("builtin"); +const assert = std.debug.assert; + +const Mir = @This(); +const bits = @import("bits.zig"); +const InternPool = @import("../../InternPool.zig"); +const Emit = @import("Emit.zig"); +const codegen = @import("../../codegen.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); + +const Instruction = bits.Instruction; +const ASI = bits.Instruction.ASI; +const Register = bits.Register; + +instructions: std.MultiArrayList(Inst).Slice, + +/// The meaning of this data is determined by `Inst.Tag` value. +extra: []const u32, + +pub const Inst = struct { + tag: Tag, + /// The meaning of this depends on `tag`. + data: Data, + + pub const Tag = enum(u16) { + /// Pseudo-instruction: End of prologue + dbg_prologue_end, + /// Pseudo-instruction: Beginning of epilogue + dbg_epilogue_begin, + /// Pseudo-instruction: Update debug line + dbg_line, + + // All the real instructions are ordered by their section number + // in The SPARC Architecture Manual, Version 9. + + /// A.2 Add + /// This uses the arithmetic_3op field. + // TODO add other operations. + add, + addcc, + + /// A.3 Branch on Integer Register with Prediction (BPr) + /// This uses the branch_predict_reg field. + bpr, + + /// A.7 Branch on Integer Condition Codes with Prediction (BPcc) + /// This uses the branch_predict_int field. + bpcc, + + /// A.8 Call and Link + /// This uses the branch_link field. + call, + + /// A.24 Jump and Link + /// This uses the arithmetic_3op field. + jmpl, + + /// A.27 Load Integer + /// This uses the arithmetic_3op field. + /// Note that the ldd variant of this instruction is deprecated, so do not emit + /// it unless specifically requested (e.g. by inline assembly). + // TODO add other operations. + ldub, + lduh, + lduw, + ldx, + + /// A.28 Load Integer from Alternate Space + /// This uses the mem_asi field. + /// Note that the ldda variant of this instruction is deprecated, so do not emit + /// it unless specifically requested (e.g. by inline assembly). + // TODO add other operations. + lduba, + lduha, + lduwa, + ldxa, + + /// A.31 Logical Operations + /// This uses the arithmetic_3op field. + // TODO add other operations. + @"and", + @"or", + xor, + xnor, + + /// A.32 Memory Barrier + /// This uses the membar_mask field. + membar, + + /// A.35 Move Integer Register on Condition (MOVcc) + /// This uses the conditional_move_int field. + movcc, + + /// A.36 Move Integer Register on Register Condition (MOVr) + /// This uses the conditional_move_reg field. + movr, + + /// A.37 Multiply and Divide (64-bit) + /// This uses the arithmetic_3op field. + mulx, + sdivx, + udivx, + + /// A.40 No Operation + /// This uses the nop field. + nop, + + /// A.45 RETURN + /// This uses the arithmetic_2op field. + @"return", + + /// A.46 SAVE and RESTORE + /// This uses the arithmetic_3op field. + save, + restore, + + /// A.48 SETHI + /// This uses the sethi field. + sethi, + + /// A.49 Shift + /// This uses the shift field. + sll, + srl, + sra, + sllx, + srlx, + srax, + + /// A.54 Store Integer + /// This uses the arithmetic_3op field. + /// Note that the std variant of this instruction is deprecated, so do not emit + /// it unless specifically requested (e.g. by inline assembly). + // TODO add other operations. + stb, + sth, + stw, + stx, + + /// A.55 Store Integer into Alternate Space + /// This uses the mem_asi field. + /// Note that the stda variant of this instruction is deprecated, so do not emit + /// it unless specifically requested (e.g. by inline assembly). + // TODO add other operations. + stba, + stha, + stwa, + stxa, + + /// A.56 Subtract + /// This uses the arithmetic_3op field. + // TODO add other operations. + sub, + subcc, + + /// A.61 Trap on Integer Condition Codes (Tcc) + /// This uses the trap field. + tcc, + + // SPARCv9 synthetic instructions + // Note that the instructions that is added here are only those that + // will simplify backend development. Synthetic instructions that is + // only used to provide syntactic sugar in, e.g. inline assembly should + // be deconstructed inside the parser instead. + // See also: G.3 Synthetic Instructions + // TODO add more synthetic instructions + + /// Comparison + /// This uses the arithmetic_2op field. + cmp, // cmp rs1, rs2/imm -> subcc rs1, rs2/imm, %g0 + + /// Copy register/immediate contents to another register + /// This uses the arithmetic_2op field, with rs1 + /// being the *destination* register. + // TODO is it okay to abuse rs1 in this way? + mov, // mov rs2/imm, rs1 -> or %g0, rs2/imm, rs1 + + /// Bitwise negation + /// This uses the arithmetic_2op field, with rs1 + /// being the *destination* register. + // TODO is it okay to abuse rs1 in this way? + // not rs2, rs1 -> xnor rs2, %g0, rs1 + // not imm, rs1 -> xnor %g0, imm, rs1 + not, + }; + + /// The position of an MIR instruction within the `Mir` instructions array. + pub const Index = u32; + + /// All instructions have a 8-byte payload, which is contained within + /// this union. `Tag` determines which union field is active, as well as + /// how to interpret the data within. + // TODO this is a quick-n-dirty solution that needs to be cleaned up. + pub const Data = union { + /// Debug info: line and column + /// + /// Used by e.g. dbg_line + dbg_line_column: struct { + line: u32, + column: u32, + }, + + /// Two operand arithmetic. + /// if is_imm true then it uses the imm field of rs2_or_imm, + /// otherwise it uses rs2 field. + /// + /// Used by e.g. return + arithmetic_2op: struct { + is_imm: bool, + rs1: Register, + rs2_or_imm: union { + rs2: Register, + imm: i13, + }, + }, + + /// Three operand arithmetic. + /// if is_imm true then it uses the imm field of rs2_or_imm, + /// otherwise it uses rs2 field. + /// + /// Used by e.g. add, sub + arithmetic_3op: struct { + is_imm: bool, + rd: Register, + rs1: Register, + rs2_or_imm: union { + rs2: Register, + imm: i13, + }, + }, + + /// Branch and link (always unconditional). + /// Used by e.g. call + branch_link: struct { + inst: Index, + // link is always %o7 + }, + + /// Branch with prediction, checking the integer status code + /// Used by e.g. bpcc + branch_predict_int: struct { + annul: bool = false, + pt: bool = true, + ccr: Instruction.CCR, + cond: Instruction.ICondition, + inst: Index, + }, + + /// Branch with prediction, comparing a register's content with zero + /// Used by e.g. bpr + branch_predict_reg: struct { + annul: bool = false, + pt: bool = true, + cond: Instruction.RCondition, + rs1: Register, + inst: Index, + }, + + /// ASI-tagged memory operations. + /// Used by e.g. ldxa, stxa + mem_asi: struct { + rd: Register, + rs1: Register, + rs2: Register = .g0, + asi: ASI, + }, + + /// Membar mask, controls the barrier behavior + /// Used by e.g. membar + membar_mask: struct { + mmask: Instruction.MemOrderingConstraint = .{}, + cmask: Instruction.MemCompletionConstraint = .{}, + }, + + /// Conditional move, checking the integer status code + /// if is_imm true then it uses the imm field of rs2_or_imm, + /// otherwise it uses rs2 field. + /// + /// Used by e.g. movcc + conditional_move_int: struct { + is_imm: bool, + ccr: Instruction.CCR, + cond: Instruction.Condition, + rd: Register, + rs2_or_imm: union { + rs2: Register, + imm: i11, + }, + }, + + /// Conditional move, comparing a register's content with zero + /// if is_imm true then it uses the imm field of rs2_or_imm, + /// otherwise it uses rs2 field. + /// + /// Used by e.g. movr + conditional_move_reg: struct { + is_imm: bool, + cond: Instruction.RCondition, + rd: Register, + rs1: Register, + rs2_or_imm: union { + rs2: Register, + imm: i10, + }, + }, + + /// No additional data + /// + /// Used by e.g. flushw + nop: void, + + /// SETHI operands. + /// + /// Used by sethi + sethi: struct { + rd: Register, + imm: u22, + }, + + /// Shift operands. + /// if is_imm true then it uses the imm field of rs2_or_imm, + /// otherwise it uses rs2 field. + /// + /// Used by e.g. sllx + shift: struct { + is_imm: bool, + rd: Register, + rs1: Register, + rs2_or_imm: union { + rs2: Register, + imm: u6, + }, + }, + + /// Trap. + /// if is_imm true then it uses the imm field of rs2_or_imm, + /// otherwise it uses rs2 field. + /// + /// Used by e.g. tcc + trap: struct { + is_imm: bool = true, + cond: Instruction.ICondition, + ccr: Instruction.CCR = .icc, + rs1: Register = .g0, + rs2_or_imm: union { + rs2: Register, + imm: u7, + }, + }, + }; + + // Make sure we don't accidentally make instructions bigger than expected. + // Note that in safety builds, Zig is allowed to insert a secret field for safety checks. + comptime { + if (!std.debug.runtime_safety) { + assert(@sizeOf(Data) == 8); + } + } +}; + +pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { + mir.instructions.deinit(gpa); + gpa.free(mir.extra); + mir.* = undefined; +} + +pub fn emit( + mir: Mir, + lf: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + atom_index: u32, + w: *std.Io.Writer, + debug_output: link.File.DebugInfoOutput, +) (codegen.CodeGenError || std.Io.Writer.Error)!void { + _ = atom_index; + const zcu = pt.zcu; + const func = zcu.funcInfo(func_index); + const nav = func.owner_nav; + const mod = zcu.navFileScope(nav).mod.?; + var e: Emit = .{ + .mir = mir, + .bin_file = lf, + .debug_output = debug_output, + .target = &mod.resolved_target.result, + .src_loc = src_loc, + .w = w, + .prev_di_pc = 0, + .prev_di_line = func.lbrace_line, + .prev_di_column = func.lbrace_column, + }; + defer e.deinit(); + e.emitMir() catch |err| switch (err) { + error.EmitFail => return zcu.codegenFailMsg(nav, e.err_msg.?), + else => |err1| return err1, + }; +} + +/// Returns the requested data, as well as the new index which is at the start of the +/// trailers for the object. +pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } { + const fields = std.meta.fields(T); + var i: usize = index; + var result: T = undefined; + inline for (fields) |field| { + @field(result, field.name) = switch (field.type) { + u32 => mir.extra[i], + i32 => @as(i32, @bitCast(mir.extra[i])), + else => @compileError("bad field type"), + }; + i += 1; + } + return .{ + .data = result, + .end = i, + }; +} diff --git a/src/codegen/sparc64/abi.zig b/src/codegen/sparc64/abi.zig new file mode 100644 index 0000000000000000000000000000000000000000..a26ab9a20b7db94620f33797b195cb9817d4467f --- /dev/null +++ b/src/codegen/sparc64/abi.zig @@ -0,0 +1,59 @@ +const std = @import("std"); +const bits = @import("bits.zig"); +const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; + +// SPARCv9 SysV ABI stack constants. +// See: Registers and the Stack Frame, page 3P-8, SCD 2.4.1. + +// The ABI specifies that %sp points to top of stack - stack bias, +// and %fp points to top of previous frame - stack bias. +pub const stack_bias = 2047; + +// The first 128 bytes of the stack is reserved for register saving purposes. +// The ABI also requires to reserve space in the stack for the first six +// outgoing arguments, even though they are usually passed in registers. +// TODO Don't allocate the argument space in leaf functions +// TODO Save an RO copy of outgoing arguments in reserved area when building in Debug +// TODO Should we also save it in ReleaseSafe? Solaris and OpenBSD binaries seem to ship +// with argument copying enabled and it doesn't seem to give them big slowdowns so +// I guess it would be okay to do in ReleaseSafe? +pub const stack_reserved_area = 128 + 48; + +// There are no callee-preserved registers since the windowing +// mechanism already takes care of them. +// We still need to preserve %o0-%o5, %g1, %g4, and %g5 before calling +// something, though, as those are shared with the callee and might be +// thrashed by it. +pub const caller_preserved_regs = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5, .g1, .g4, .g5 }; + +// Try to allocate i, l, o, then g sets of registers, in order of priority. +const allocatable_regs = [_]Register{ + // zig fmt: off + .@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5", + .l0, .l1, .l2, .l3, .l4, .l5, .l6, .l7, + .o0, .o1, .o2, .o3, .o4, .o5, + .g1, .g4, .g5, + // zig fmt: on +}; + +pub const c_abi_int_param_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5 }; +pub const c_abi_int_param_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5" }; + +pub const c_abi_int_return_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3 }; +pub const c_abi_int_return_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3" }; + +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = 0, + .end = allocatable_regs.len, + }, true); + break :blk set; + }; +}; diff --git a/src/codegen/sparc64/bits.zig b/src/codegen/sparc64/bits.zig new file mode 100644 index 0000000000000000000000000000000000000000..d27371bf2505f2c7a4abde9255834f3063e896c2 --- /dev/null +++ b/src/codegen/sparc64/bits.zig @@ -0,0 +1,1621 @@ +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; + +/// General purpose registers in the SPARCv9 instruction set +pub const Register = enum(u6) { + // zig fmt: off + g0, g1, g2, g3, g4, g5, g6, g7, + o0, o1, o2, o3, o4, o5, o6, o7, + l0, l1, l2, l3, l4, l5, l6, l7, + @"i0", @"i1", @"i2", @"i3", @"i4", @"i5", @"i6", @"i7", + + sp = 46, // stack pointer (o6) + fp = 62, // frame pointer (i6) + // zig fmt: on + + pub fn id(reg: Register) u5 { + return @truncate(@intFromEnum(reg)); + } + + pub fn enc(reg: Register) u5 { + // For integer registers, enc() == id(). + return reg.id(); + } + + pub fn dwarfNum(reg: Register) u5 { + return reg.id(); + } +}; + +test "Register.id" { + // SP + try testing.expectEqual(@as(u5, 14), Register.o6.id()); + try testing.expectEqual(Register.o6.id(), Register.sp.id()); + + // FP + try testing.expectEqual(@as(u5, 30), Register.i6.id()); + try testing.expectEqual(Register.i6.id(), Register.fp.id()); + + // x0 + try testing.expectEqual(@as(u5, 0), Register.g0.id()); + try testing.expectEqual(@as(u5, 8), Register.o0.id()); + try testing.expectEqual(@as(u5, 16), Register.l0.id()); + try testing.expectEqual(@as(u5, 24), Register.i0.id()); +} + +test "Register.enc" { + // x0 + try testing.expectEqual(@as(u5, 0), Register.g0.enc()); + try testing.expectEqual(@as(u5, 8), Register.o0.enc()); + try testing.expectEqual(@as(u5, 16), Register.l0.enc()); + try testing.expectEqual(@as(u5, 24), Register.i0.enc()); + + // For integer registers, enc() == id(). + try testing.expectEqual(Register.g0.enc(), Register.g0.id()); + try testing.expectEqual(Register.o0.enc(), Register.o0.id()); + try testing.expectEqual(Register.l0.enc(), Register.l0.id()); + try testing.expectEqual(Register.i0.enc(), Register.i0.id()); +} + +/// Scalar floating point registers in the SPARCv9 instruction set +pub const FloatingPointRegister = enum(u7) { + // SPARCv9 has 64 f32 registers, 32 f64 registers, and 16 f128 registers, + // which are aliased in this way: + // + // | %d0 | %d2 | + // %q0 | %f0 | %f1 | %f2 | %f3 | + // | %d4 | %d6 | + // %q4 | %f4 | %f5 | %f6 | %f7 | + // ... + // | %d60 | %d62 | + // %q60 | %f60 | %f61 | %f62 | %f63 | + // + // Though, since the instructions uses five-bit addressing, only %f0-%f31 + // is usable with f32 instructions. + + // zig fmt: off + + // 32-bit registers + @"f0", @"f1", @"f2", @"f3", @"f4", @"f5", @"f6", @"f7", + @"f8", @"f9", @"f10", @"f11", @"f12", @"f13", @"f14", @"f15", + @"f16", @"f17", @"f18", @"f19", @"f20", @"f21", @"f22", @"f23", + @"f24", @"f25", @"f26", @"f27", @"f28", @"f29", @"f30", @"f31", + + // 64-bit registers + d0, d2, d4, d6, d8, d10, d12, d14, + d16, d18, d20, d22, d24, d26, d28, d30, + d32, d34, d36, d38, d40, d42, d44, d46, + d48, d50, d52, d54, d56, d58, d60, d62, + + // 128-bit registers + q0, q4, q8, q12, q16, q20, q24, q28, + q32, q36, q40, q44, q48, q52, q56, q60, + // zig fmt: on + + pub fn id(self: FloatingPointRegister) u6 { + return switch (self.size()) { + 32 => @as(u6, @truncate(@intFromEnum(self))), + 64 => @as(u6, @truncate((@intFromEnum(self) - 32) * 2)), + 128 => @as(u6, @truncate((@intFromEnum(self) - 64) * 4)), + else => unreachable, + }; + } + + pub fn enc(self: FloatingPointRegister) u5 { + // Floating point registers use an encoding scheme to map from the 6-bit + // ID to 5-bit encoded value. + // (See section 5.1.4.1 of SPARCv9 ISA specification) + + const reg_id = self.id(); + return @as(u5, @truncate(reg_id | (reg_id >> 5))); + } + + /// Returns the bit-width of the register. + pub fn size(self: FloatingPointRegister) u8 { + return switch (@intFromEnum(self)) { + 0...31 => 32, + 32...63 => 64, + 64...79 => 128, + else => unreachable, + }; + } +}; + +test "FloatingPointRegister.id" { + // Low region + try testing.expectEqual(@as(u6, 0), FloatingPointRegister.q0.id()); + try testing.expectEqual(FloatingPointRegister.q0.id(), FloatingPointRegister.d0.id()); + try testing.expectEqual(FloatingPointRegister.d0.id(), FloatingPointRegister.f0.id()); + + try testing.expectEqual(@as(u6, 28), FloatingPointRegister.q28.id()); + try testing.expectEqual(FloatingPointRegister.q28.id(), FloatingPointRegister.d28.id()); + try testing.expectEqual(FloatingPointRegister.d28.id(), FloatingPointRegister.f28.id()); + + // High region + try testing.expectEqual(@as(u6, 32), FloatingPointRegister.q32.id()); + try testing.expectEqual(FloatingPointRegister.q32.id(), FloatingPointRegister.d32.id()); + + try testing.expectEqual(@as(u6, 60), FloatingPointRegister.q60.id()); + try testing.expectEqual(FloatingPointRegister.q60.id(), FloatingPointRegister.d60.id()); +} + +test "FloatingPointRegister.enc" { + // f registers + try testing.expectEqual(@as(u5, 0), FloatingPointRegister.f0.enc()); + try testing.expectEqual(@as(u5, 1), FloatingPointRegister.f1.enc()); + try testing.expectEqual(@as(u5, 31), FloatingPointRegister.f31.enc()); + + // d registers + try testing.expectEqual(@as(u5, 0), FloatingPointRegister.d0.enc()); + try testing.expectEqual(@as(u5, 1), FloatingPointRegister.d32.enc()); + try testing.expectEqual(@as(u5, 31), FloatingPointRegister.d62.enc()); + + // q registers + try testing.expectEqual(@as(u5, 0), FloatingPointRegister.q0.enc()); + try testing.expectEqual(@as(u5, 1), FloatingPointRegister.q32.enc()); + try testing.expectEqual(@as(u5, 29), FloatingPointRegister.q60.enc()); +} + +/// Represents an instruction in the SPARCv9 instruction set +pub const Instruction = union(enum) { + // Some of the instruction formats have several minor formats, here I + // name them with letters since there's no official naming scheme. + // TODO: need to rename the minor formats to a more descriptive name. + + // I am using regular structs instead of packed ones to avoid + // endianness-dependent behavior when constructing the actual + // assembly instructions. + // See also: https://github.com/ziglang/zig/issues/10113 + // TODO: change it back to packed structs once the issue is resolved. + + // Format 1 (op = 1): CALL + format_1: struct { + op: u2 = 0b01, + disp30: u30, + }, + + // Format 2 (op = 0): SETHI & Branches (Bicc, BPcc, BPr, FBfcc, FBPfcc) + format_2a: struct { + op: u2 = 0b00, + rd: u5, + op2: u3, + imm22: u22, + }, + format_2b: struct { + op: u2 = 0b00, + a: u1, + cond: u4, + op2: u3, + disp22: u22, + }, + format_2c: struct { + op: u2 = 0b00, + a: u1, + cond: u4, + op2: u3, + cc1: u1, + cc0: u1, + p: u1, + disp19: u19, + }, + format_2d: struct { + op: u2 = 0b00, + a: u1, + fixed: u1 = 0b0, + rcond: u3, + op2: u3, + d16hi: u2, + p: u1, + rs1: u5, + d16lo: u14, + }, + + // Format 3 (op = 2 or 3): Arithmetic, Logical, MOVr, MEMBAR, Load, and Store + format_3a: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b0, + reserved: u8 = 0b00000000, + rs2: u5, + }, + format_3b: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b1, + simm13: u13, + }, + format_3c: struct { + op: u2, + reserved1: u5 = 0b00000, + op3: u6, + rs1: u5, + i: u1 = 0b0, + reserved2: u8 = 0b00000000, + rs2: u5, + }, + format_3d: struct { + op: u2, + reserved: u5 = 0b00000, + op3: u6, + rs1: u5, + i: u1 = 0b1, + simm13: u13, + }, + format_3e: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b0, + rcond: u3, + reserved: u5 = 0b00000, + rs2: u5, + }, + format_3f: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b1, + rcond: u3, + simm10: u10, + }, + format_3g: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + // See Errata 58 of SPARCv9 specification + // https://sparc.org/errata-for-v9/#58 + i: u1 = 0b1, + reserved: u8 = 0b00000000, + rs2: u5, + }, + format_3h: struct { + op: u2 = 0b10, + fixed1: u5 = 0b00000, + op3: u6 = 0b101000, + fixed2: u5 = 0b01111, + i: u1 = 0b1, + reserved: u6 = 0b000000, + cmask: u3, + mmask: u4, + }, + format_3i: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b0, + imm_asi: u8, + rs2: u5, + }, + format_3j: struct { + op: u2, + impl_dep1: u5, + op3: u6, + impl_dep2: u19, + }, + format_3k: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b0, + x: u1, + reserved: u7 = 0b0000000, + rs2: u5, + }, + format_3l: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b1, + x: u1 = 0b0, + reserved: u7 = 0b0000000, + shcnt32: u5, + }, + format_3m: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b1, + x: u1 = 0b1, + reserved: u6 = 0b000000, + shcnt64: u6, + }, + format_3n: struct { + op: u2, + rd: u5, + op3: u6, + reserved: u5 = 0b00000, + opf: u9, + rs2: u5, + }, + format_3o: struct { + op: u2, + fixed: u3 = 0b000, + cc1: u1, + cc0: u1, + op3: u6, + rs1: u5, + opf: u9, + rs2: u5, + }, + format_3p: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + opf: u9, + rs2: u5, + }, + format_3q: struct { + op: u2, + rd: u5, + op3: u6, + rs1: u5, + reserved: u14 = 0b00000000000000, + }, + format_3r: struct { + op: u2, + fcn: u5, + op3: u6, + reserved: u19 = 0b0000000000000000000, + }, + format_3s: struct { + op: u2, + rd: u5, + op3: u6, + reserved: u19 = 0b0000000000000000000, + }, + + //Format 4 (op = 2): MOVcc, FMOVr, FMOVcc, and Tcc + format_4a: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b0, + cc1: u1, + cc0: u1, + reserved: u6 = 0b000000, + rs2: u5, + }, + format_4b: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b1, + cc1: u1, + cc0: u1, + simm11: u11, + }, + format_4c: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + cc2: u1, + cond: u4, + i: u1 = 0b0, + cc1: u1, + cc0: u1, + reserved: u6 = 0b000000, + rs2: u5, + }, + format_4d: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + cc2: u1, + cond: u4, + i: u1 = 0b1, + cc1: u1, + cc0: u1, + simm11: u11, + }, + format_4e: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + rs1: u5, + i: u1 = 0b1, + cc1: u1, + cc0: u1, + reserved: u4 = 0b0000, + sw_trap: u7, + }, + format_4f: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + rs1: u5, + fixed: u1 = 0b0, + rcond: u3, + opf_low: u5, + rs2: u5, + }, + format_4g: struct { + op: u2 = 0b10, + rd: u5, + op3: u6, + fixed: u1 = 0b0, + cond: u4, + opf_cc: u3, + opf_low: u6, + rs2: u5, + }, + + pub const CCR = enum(u3) { + fcc0, + fcc1, + fcc2, + fcc3, + icc, + reserved1, + xcc, + reserved2, + }; + + pub const RCondition = enum(u3) { + reserved1, + eq_zero, + le_zero, + lt_zero, + reserved2, + ne_zero, + gt_zero, + ge_zero, + + /// Returns the condition which is true iff the given condition is + /// false (if such a condition exists). + pub fn negate(cond: RCondition) RCondition { + return switch (cond) { + .eq_zero => .ne_zero, + .ne_zero => .eq_zero, + .lt_zero => .ge_zero, + .ge_zero => .lt_zero, + .le_zero => .gt_zero, + .gt_zero => .le_zero, + .reserved1 => unreachable, + .reserved2 => unreachable, + }; + } + }; + + pub const ASI = enum(u8) { + asi_nucleus = 0x04, + asi_nucleus_little = 0x0c, + asi_as_if_user_primary = 0x10, + asi_as_if_user_secondary = 0x11, + asi_as_if_user_primary_little = 0x18, + asi_as_if_user_secondary_little = 0x19, + asi_primary = 0x80, + asi_secondary = 0x81, + asi_primary_nofault = 0x82, + asi_secondary_nofault = 0x83, + asi_primary_little = 0x88, + asi_secondary_little = 0x89, + asi_primary_nofault_little = 0x8a, + asi_secondary_nofault_little = 0x8b, + }; + + pub const ShiftWidth = enum(u1) { + shift32, + shift64, + }; + + pub const MemOrderingConstraint = packed struct { + store_store: bool = false, + load_store: bool = false, + store_load: bool = false, + load_load: bool = false, + }; + + pub const MemCompletionConstraint = packed struct { + sync: bool = false, + mem_issue: bool = false, + lookaside: bool = false, + }; + + // In SPARCv9, FP and integer comparison operations + // are encoded differently. + + pub const FCondition = enum(u4) { + /// Branch Never + nv, + /// Branch on Not Equal + ne, + /// Branch on Less or Greater + lg, + /// Branch on Unordered or Less + ul, + /// Branch on Less + lt, + /// Branch on Unordered or Greater + ug, + /// Branch on Greater + gt, + /// Branch on Unordered + un, + /// Branch Always + al, + /// Branch on Equal + eq, + /// Branch on Unordered or Equal + ue, + /// Branch on Greater or Equal + ge, + /// Branch on Unordered or Greater or Equal + uge, + /// Branch on Less or Equal + le, + /// Branch on Unordered or Less or Equal + ule, + /// Branch on Ordered + ord, + + /// Converts a std.math.CompareOperator into a condition flag, + /// i.e. returns the condition that is true iff the result of the + /// comparison is true. + pub fn fromCompareOperator(op: std.math.CompareOperator) FCondition { + return switch (op) { + .gte => .ge, + .gt => .gt, + .neq => .ne, + .lt => .lt, + .lte => .le, + .eq => .eq, + }; + } + + /// Returns the condition which is true iff the given condition is + /// false (if such a condition exists). + pub fn negate(cond: FCondition) FCondition { + return switch (cond) { + .eq => .ne, + .ne => .eq, + .ge => .ul, + .ul => .ge, + .le => .ug, + .ug => .le, + .lt => .uge, + .uge => .lt, + .gt => .ule, + .ule => .gt, + .ue => .lg, + .lg => .ue, + .ord => .un, + .un => .ord, + .al => unreachable, + .nv => unreachable, + }; + } + }; + + pub const ICondition = enum(u4) { + /// Branch Never + nv, + /// Branch on Equal + eq, + /// Branch on Less or Equal + le, + /// Branch on Less + lt, + /// Branch on Less or Equal Unsigned + leu, + /// Branch on Carry Set (Less than, Unsigned) + cs, + /// Branch on Negative + neg, + /// Branch on Overflow Set + vs, + /// Branch Always + al, + /// Branch on Not Equal + ne, + /// Branch on Greater + gt, + /// Branch on Greater or Equal + ge, + /// Branch on Greater Unsigned + gu, + /// Branch on Carry Clear (Greater Than or Equal, Unsigned) + cc, + /// Branch on Positive + pos, + /// Branch on Overflow Clear + vc, + + /// Converts a std.math.CompareOperator into a condition flag, + /// i.e. returns the condition that is true iff the result of the + /// comparison is true. Assumes signed comparison. + pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) ICondition { + return switch (op) { + .gte => .ge, + .gt => .gt, + .neq => .ne, + .lt => .lt, + .lte => .le, + .eq => .eq, + }; + } + + /// Converts a std.math.CompareOperator into a condition flag, + /// i.e. returns the condition that is true iff the result of the + /// comparison is true. Assumes unsigned comparison. + pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) ICondition { + return switch (op) { + .gte => .cc, + .gt => .gu, + .neq => .ne, + .lt => .cs, + .lte => .leu, + .eq => .eq, + }; + } + + /// Returns the condition which is true iff the given condition is + /// false (if such a condition exists). + pub fn negate(cond: ICondition) ICondition { + return switch (cond) { + .eq => .ne, + .ne => .eq, + .cs => .cc, + .cc => .cs, + .neg => .pos, + .pos => .neg, + .vs => .vc, + .vc => .vs, + .gu => .leu, + .leu => .gu, + .ge => .lt, + .lt => .ge, + .gt => .le, + .le => .gt, + .al => unreachable, + .nv => unreachable, + }; + } + }; + + pub const ConditionTag = enum { fcond, icond }; + pub const Condition = union(ConditionTag) { + fcond: FCondition, + icond: ICondition, + + /// Encodes the condition into the instruction bit pattern. + pub fn enc(cond: Condition) u4 { + return switch (cond) { + .icond => |c| @intFromEnum(c), + .fcond => |c| @intFromEnum(c), + }; + } + + /// Returns the condition which is true iff the given condition is + /// false (if such a condition exists). + pub fn negate(cond: Condition) Condition { + return switch (cond) { + .icond => |c| .{ .icond = c.negate() }, + .fcond => |c| .{ .fcond = c.negate() }, + }; + } + }; + + pub fn toU32(self: Instruction) u32 { + // TODO: Remove this once packed structs work. + return switch (self) { + .format_1 => |v| (@as(u32, v.op) << 30) | @as(u32, v.disp30), + .format_2a => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op2) << 22) | @as(u32, v.imm22), + .format_2b => |v| (@as(u32, v.op) << 30) | (@as(u32, v.a) << 29) | (@as(u32, v.cond) << 25) | (@as(u32, v.op2) << 22) | @as(u32, v.disp22), + .format_2c => |v| (@as(u32, v.op) << 30) | (@as(u32, v.a) << 29) | (@as(u32, v.cond) << 25) | (@as(u32, v.op2) << 22) | (@as(u32, v.cc1) << 21) | (@as(u32, v.cc0) << 20) | (@as(u32, v.p) << 19) | @as(u32, v.disp19), + .format_2d => |v| (@as(u32, v.op) << 30) | (@as(u32, v.a) << 29) | (@as(u32, v.fixed) << 28) | (@as(u32, v.rcond) << 25) | (@as(u32, v.op2) << 22) | (@as(u32, v.d16hi) << 20) | (@as(u32, v.p) << 19) | (@as(u32, v.rs1) << 14) | @as(u32, v.d16lo), + .format_3a => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), + .format_3b => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | @as(u32, v.simm13), + .format_3c => |v| (@as(u32, v.op) << 30) | (@as(u32, v.reserved1) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved2) << 5) | @as(u32, v.rs2), + .format_3d => |v| (@as(u32, v.op) << 30) | (@as(u32, v.reserved) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | @as(u32, v.simm13), + .format_3e => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.rcond) << 10) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), + .format_3f => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.rcond) << 10) | @as(u32, v.simm10), + .format_3g => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), + .format_3h => |v| (@as(u32, v.op) << 30) | (@as(u32, v.fixed1) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.fixed2) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.reserved) << 7) | (@as(u32, v.cmask) << 4) | @as(u32, v.mmask), + .format_3i => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.imm_asi) << 5) | @as(u32, v.rs2), + .format_3j => |v| (@as(u32, v.op) << 30) | (@as(u32, v.impl_dep1) << 25) | (@as(u32, v.op3) << 19) | @as(u32, v.impl_dep2), + .format_3k => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.x) << 12) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), + .format_3l => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.x) << 12) | (@as(u32, v.reserved) << 5) | @as(u32, v.shcnt32), + .format_3m => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.x) << 12) | (@as(u32, v.reserved) << 6) | @as(u32, v.shcnt64), + .format_3n => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.reserved) << 14) | (@as(u32, v.opf) << 5) | @as(u32, v.rs2), + .format_3o => |v| (@as(u32, v.op) << 30) | (@as(u32, v.fixed) << 27) | (@as(u32, v.cc1) << 26) | (@as(u32, v.cc0) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.opf) << 5) | @as(u32, v.rs2), + .format_3p => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.opf) << 5) | @as(u32, v.rs2), + .format_3q => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | @as(u32, v.reserved), + .format_3r => |v| (@as(u32, v.op) << 30) | (@as(u32, v.fcn) << 25) | (@as(u32, v.op3) << 19) | @as(u32, v.reserved), + .format_3s => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | @as(u32, v.reserved), + .format_4a => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), + .format_4b => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | @as(u32, v.simm11), + .format_4c => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.cc2) << 18) | (@as(u32, v.cond) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | (@as(u32, v.reserved) << 5) | @as(u32, v.rs2), + .format_4d => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.cc2) << 18) | (@as(u32, v.cond) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | @as(u32, v.simm11), + .format_4e => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.i) << 13) | (@as(u32, v.cc1) << 12) | (@as(u32, v.cc0) << 11) | (@as(u32, v.reserved) << 7) | @as(u32, v.sw_trap), + .format_4f => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.rs1) << 14) | (@as(u32, v.fixed) << 13) | (@as(u32, v.rcond) << 10) | (@as(u32, v.opf_low) << 5) | @as(u32, v.rs2), + .format_4g => |v| (@as(u32, v.op) << 30) | (@as(u32, v.rd) << 25) | (@as(u32, v.op3) << 19) | (@as(u32, v.fixed) << 18) | (@as(u32, v.cond) << 14) | (@as(u32, v.opf_cc) << 11) | (@as(u32, v.opf_low) << 5) | @as(u32, v.rs2), + }; + } + + // SPARCv9 Instruction formats. + // See section 6.2 of the SPARCv9 ISA manual. + + fn format1(disp: i32) Instruction { + const udisp = @as(u32, @bitCast(disp)); + + // In SPARC, branch target needs to be aligned to 4 bytes. + assert(udisp % 4 == 0); + + // Discard the last two bits since those are implicitly zero. + const udisp_truncated = @as(u30, @truncate(udisp >> 2)); + return Instruction{ + .format_1 = .{ + .disp30 = udisp_truncated, + }, + }; + } + + fn format2a(op2: u3, imm: u22, rd: Register) Instruction { + return Instruction{ + .format_2a = .{ + .rd = rd.enc(), + .op2 = op2, + .imm22 = imm, + }, + }; + } + + fn format2b(op2: u3, cond: Condition, annul: bool, disp: i24) Instruction { + const udisp = @as(u24, @bitCast(disp)); + + // In SPARC, branch target needs to be aligned to 4 bytes. + assert(udisp % 4 == 0); + + // Discard the last two bits since those are implicitly zero. + const udisp_truncated = @as(u22, @truncate(udisp >> 2)); + return Instruction{ + .format_2b = .{ + .a = @intFromBool(annul), + .cond = cond.enc(), + .op2 = op2, + .disp22 = udisp_truncated, + }, + }; + } + + fn format2c(op2: u3, cond: Condition, annul: bool, pt: bool, ccr: CCR, disp: i21) Instruction { + const udisp = @as(u21, @bitCast(disp)); + + // In SPARC, branch target needs to be aligned to 4 bytes. + assert(udisp % 4 == 0); + + // Discard the last two bits since those are implicitly zero. + const udisp_truncated = @as(u19, @truncate(udisp >> 2)); + + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_2c = .{ + .a = @intFromBool(annul), + .cond = cond.enc(), + .op2 = op2, + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .p = @intFromBool(pt), + .disp19 = udisp_truncated, + }, + }; + } + + fn format2d(op2: u3, rcond: RCondition, annul: bool, pt: bool, rs1: Register, disp: i18) Instruction { + const udisp = @as(u18, @bitCast(disp)); + + // In SPARC, branch target needs to be aligned to 4 bytes. + assert(udisp % 4 == 0); + + // Discard the last two bits since those are implicitly zero, + // and split it into low and high parts. + const udisp_truncated = @as(u16, @truncate(udisp >> 2)); + const udisp_hi = @as(u2, @truncate((udisp_truncated & 0b1100_0000_0000_0000) >> 14)); + const udisp_lo = @as(u14, @truncate(udisp_truncated & 0b0011_1111_1111_1111)); + return Instruction{ + .format_2d = .{ + .a = @intFromBool(annul), + .rcond = @intFromEnum(rcond), + .op2 = op2, + .p = @intFromBool(pt), + .rs1 = rs1.enc(), + .d16hi = udisp_hi, + .d16lo = udisp_lo, + }, + }; + } + + fn format3a(op: u2, op3: u6, rs1: Register, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_3a = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .rs2 = rs2.enc(), + }, + }; + } + fn format3b(op: u2, op3: u6, rs1: Register, imm: i13, rd: Register) Instruction { + return Instruction{ + .format_3b = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .simm13 = @as(u13, @bitCast(imm)), + }, + }; + } + fn format3c(op: u2, op3: u6, rs1: Register, rs2: Register) Instruction { + return Instruction{ + .format_3c = .{ + .op = op, + .op3 = op3, + .rs1 = rs1.enc(), + .rs2 = rs2.enc(), + }, + }; + } + fn format3d(op: u2, op3: u6, rs1: Register, imm: i13) Instruction { + return Instruction{ + .format_3d = .{ + .op = op, + .op3 = op3, + .rs1 = rs1.enc(), + .simm13 = @as(u13, @bitCast(imm)), + }, + }; + } + fn format3e(op: u2, op3: u6, rcond: RCondition, rs1: Register, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_3e = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .rcond = @intFromEnum(rcond), + .rs2 = rs2.enc(), + }, + }; + } + fn format3f(op: u2, op3: u6, rcond: RCondition, rs1: Register, imm: i10, rd: Register) Instruction { + return Instruction{ + .format_3f = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .rcond = @intFromEnum(rcond), + .simm10 = @as(u10, @bitCast(imm)), + }, + }; + } + fn format3g(op: u2, op3: u6, rs1: Register, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_3g = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .rs2 = rs2.enc(), + }, + }; + } + fn format3h(cmask: MemCompletionConstraint, mmask: MemOrderingConstraint) Instruction { + return Instruction{ + .format_3h = .{ + .cmask = @as(u3, @bitCast(cmask)), + .mmask = @as(u4, @bitCast(mmask)), + }, + }; + } + fn format3i(op: u2, op3: u6, rs1: Register, rs2: Register, rd: Register, asi: ASI) Instruction { + return Instruction{ + .format_3i = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .imm_asi = @intFromEnum(asi), + .rs2 = rs2.enc(), + }, + }; + } + fn format3j(op: u2, op3: u6, impl_dep1: u5, impl_dep2: u19) Instruction { + return Instruction{ + .format_3j = .{ + .op = op, + .impl_dep1 = impl_dep1, + .op3 = op3, + .impl_dep2 = impl_dep2, + }, + }; + } + fn format3k(op: u2, op3: u6, sw: ShiftWidth, rs1: Register, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_3k = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .x = @intFromEnum(sw), + .rs2 = rs2.enc(), + }, + }; + } + fn format3l(op: u2, op3: u6, rs1: Register, shift_count: u5, rd: Register) Instruction { + return Instruction{ + .format_3l = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .shcnt32 = shift_count, + }, + }; + } + fn format3m(op: u2, op3: u6, rs1: Register, shift_count: u6, rd: Register) Instruction { + return Instruction{ + .format_3m = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .shcnt64 = shift_count, + }, + }; + } + fn format3n(op: u2, op3: u6, opf: u9, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_3n = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .opf = opf, + .rs2 = rs2.enc(), + }, + }; + } + fn format3o(op: u2, op3: u6, opf: u9, ccr: CCR, rs1: Register, rs2: Register) Instruction { + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_3o = .{ + .op = op, + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .op3 = op3, + .rs1 = rs1.enc(), + .opf = opf, + .rs2 = rs2.enc(), + }, + }; + } + fn format3p(op: u2, op3: u6, opf: u9, rs1: Register, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_3p = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .opf = opf, + .rs2 = rs2.enc(), + }, + }; + } + fn format3q(op: u2, op3: u6, rs1: Register, rd: Register) Instruction { + return Instruction{ + .format_3q = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + }, + }; + } + fn format3r(op: u2, op3: u6, fcn: u5) Instruction { + return Instruction{ + .format_3r = .{ + .op = op, + .fcn = fcn, + .op3 = op3, + }, + }; + } + fn format3s(op: u2, op3: u6, rd: Register) Instruction { + return Instruction{ + .format_3s = .{ + .op = op, + .rd = rd.enc(), + .op3 = op3, + }, + }; + } + + fn format4a(op3: u6, ccr: CCR, rs1: Register, rs2: Register, rd: Register) Instruction { + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_4a = .{ + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .rs2 = rs2.enc(), + }, + }; + } + + fn format4b(op3: u6, ccr: CCR, rs1: Register, imm: i11, rd: Register) Instruction { + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_4b = .{ + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .simm11 = @as(u11, @bitCast(imm)), + }, + }; + } + + fn format4c(op3: u6, cond: Condition, ccr: CCR, rs2: Register, rd: Register) Instruction { + const ccr_cc2 = @as(u1, @truncate(@intFromEnum(ccr) >> 2)); + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_4c = .{ + .rd = rd.enc(), + .op3 = op3, + .cc2 = ccr_cc2, + .cond = cond.enc(), + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .rs2 = rs2.enc(), + }, + }; + } + + fn format4d(op3: u6, cond: Condition, ccr: CCR, imm: i11, rd: Register) Instruction { + const ccr_cc2 = @as(u1, @truncate(@intFromEnum(ccr) >> 2)); + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_4d = .{ + .rd = rd.enc(), + .op3 = op3, + .cc2 = ccr_cc2, + .cond = cond.enc(), + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .simm11 = @as(u11, @bitCast(imm)), + }, + }; + } + + fn format4e(op3: u6, ccr: CCR, rs1: Register, rd: Register, sw_trap: u7) Instruction { + const ccr_cc1 = @as(u1, @truncate(@intFromEnum(ccr) >> 1)); + const ccr_cc0 = @as(u1, @truncate(@intFromEnum(ccr))); + return Instruction{ + .format_4e = .{ + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .cc1 = ccr_cc1, + .cc0 = ccr_cc0, + .sw_trap = sw_trap, + }, + }; + } + + fn format4f( + op3: u6, + opf_low: u5, + rcond: RCondition, + rs1: Register, + rs2: Register, + rd: Register, + ) Instruction { + return Instruction{ + .format_4f = .{ + .rd = rd.enc(), + .op3 = op3, + .rs1 = rs1.enc(), + .rcond = @intFromEnum(rcond), + .opf_low = opf_low, + .rs2 = rs2.enc(), + }, + }; + } + + fn format4g(op3: u6, opf_low: u6, opf_cc: u3, cond: Condition, rs2: Register, rd: Register) Instruction { + return Instruction{ + .format_4g = .{ + .rd = rd.enc(), + .op3 = op3, + .cond = cond.enc(), + .opf_cc = opf_cc, + .opf_low = opf_low, + .rs2 = rs2.enc(), + }, + }; + } + + // SPARCv9 Instruction definition. + // See appendix A of the SPARCv9 ISA manual. + + pub fn add(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_0000, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_0000, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn addcc(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b01_0000, rs1, rs2, rd), + i13 => format3b(0b10, 0b01_0000, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn bpcc(cond: ICondition, annul: bool, pt: bool, ccr: CCR, disp: i21) Instruction { + return format2c(0b001, .{ .icond = cond }, annul, pt, ccr, disp); + } + + pub fn bpr(cond: RCondition, annul: bool, pt: bool, rs1: Register, disp: i18) Instruction { + return format2d(0b011, cond, annul, pt, rs1, disp); + } + + pub fn jmpl(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b11_1000, rs1, rs2, rd), + i13 => format3b(0b10, 0b11_1000, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn ldub(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_0001, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_0001, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn lduh(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_0010, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_0010, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn lduw(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_0000, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_0000, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn ldx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_1011, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_1011, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn lduba(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_0001, rs1, rs2, rd, asi); + } + + pub fn lduha(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_0010, rs1, rs2, rd, asi); + } + + pub fn lduwa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_0000, rs1, rs2, rd, asi); + } + + pub fn ldxa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_1011, rs1, rs2, rd, asi); + } + + pub fn @"and"(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_0001, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_0001, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn @"or"(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_0010, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_0010, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn xor(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_0011, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_0011, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn xnor(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_0111, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_0111, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn membar(cmask: MemCompletionConstraint, mmask: MemOrderingConstraint) Instruction { + return format3h(cmask, mmask); + } + + pub fn movcc(comptime s2: type, cond: Condition, ccr: CCR, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format4c(0b10_1100, cond, ccr, rs2, rd), + i11 => format4d(0b10_1100, cond, ccr, rs2, rd), + else => unreachable, + }; + } + + pub fn movr(comptime s2: type, cond: RCondition, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3e(0b10, 0b10_1111, cond, rs1, rs2, rd), + i10 => format3f(0b10, 0b10_1111, cond, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn mulx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_1001, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_1001, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn sdivx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b10_1101, rs1, rs2, rd), + i13 => format3b(0b10, 0b10_1101, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn udivx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_1101, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_1101, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn nop() Instruction { + return sethi(0, .g0); + } + + pub fn @"return"(comptime s2: type, rs1: Register, rs2: s2) Instruction { + return switch (s2) { + Register => format3c(0b10, 0b11_1001, rs1, rs2), + i13 => format3d(0b10, 0b11_1001, rs1, rs2), + else => unreachable, + }; + } + + pub fn save(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b11_1100, rs1, rs2, rd), + i13 => format3b(0b10, 0b11_1100, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn restore(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b11_1101, rs1, rs2, rd), + i13 => format3b(0b10, 0b11_1101, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn sethi(imm: u22, rd: Register) Instruction { + return format2a(0b100, imm, rd); + } + + pub fn sll(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3k(0b10, 0b10_0101, .shift32, rs1, rs2, rd), + u5 => format3l(0b10, 0b10_0101, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn srl(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3k(0b10, 0b10_0110, .shift32, rs1, rs2, rd), + u5 => format3l(0b10, 0b10_0110, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn sra(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3k(0b10, 0b10_0111, .shift32, rs1, rs2, rd), + u5 => format3l(0b10, 0b10_0111, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn sllx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3k(0b10, 0b10_0101, .shift64, rs1, rs2, rd), + u6 => format3m(0b10, 0b10_0101, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn srlx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3k(0b10, 0b10_0110, .shift64, rs1, rs2, rd), + u6 => format3m(0b10, 0b10_0110, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn srax(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3k(0b10, 0b10_0111, .shift64, rs1, rs2, rd), + u6 => format3m(0b10, 0b10_0111, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn stb(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_0101, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_0101, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn sth(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_0110, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_0110, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn stw(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_0100, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_0100, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn stx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b11, 0b00_1110, rs1, rs2, rd), + i13 => format3b(0b11, 0b00_1110, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn stba(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_0101, rs1, rs2, rd, asi); + } + + pub fn stha(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_0110, rs1, rs2, rd, asi); + } + + pub fn stwa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_0100, rs1, rs2, rd, asi); + } + + pub fn stxa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction { + return format3i(0b11, 0b01_1110, rs1, rs2, rd, asi); + } + + pub fn sub(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b00_0100, rs1, rs2, rd), + i13 => format3b(0b10, 0b00_0100, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn subcc(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction { + return switch (s2) { + Register => format3a(0b10, 0b01_0100, rs1, rs2, rd), + i13 => format3b(0b10, 0b01_0100, rs1, rs2, rd), + else => unreachable, + }; + } + + pub fn trap(comptime s2: type, cond: ICondition, ccr: CCR, rs1: Register, rs2: s2) Instruction { + // Tcc instructions abuse the rd field to store the conditionals. + return switch (s2) { + Register => format4a(0b11_1010, ccr, rs1, rs2, @as(Register, @enumFromInt(@intFromEnum(cond)))), + u7 => format4e(0b11_1010, ccr, rs1, @as(Register, @enumFromInt(@intFromEnum(cond))), rs2), + else => unreachable, + }; + } +}; + +test "Serialize formats" { + const Testcase = struct { + inst: Instruction, + expected: u32, + }; + + // Note that the testcases might or might not be a valid instruction + // This is mostly just to check the behavior of the format packed structs + // since currently stage1 doesn't properly implement it in all cases + const testcases = [_]Testcase{ + .{ + .inst = Instruction.format1(4), + .expected = 0b01_000000000000000000000000000001, + }, + .{ + .inst = Instruction.format2a(4, 0, .g0), + .expected = 0b00_00000_100_0000000000000000000000, + }, + .{ + .inst = Instruction.format2b(6, .{ .icond = .lt }, true, -4), + .expected = 0b00_1_0011_110_1111111111111111111111, + }, + .{ + .inst = Instruction.format2c(3, .{ .icond = .nv }, false, true, .xcc, 8), + .expected = 0b00_0_0000_011_1_0_1_0000000000000000010, + }, + .{ + .inst = Instruction.format2d(7, .eq_zero, false, true, .o0, 20), + .expected = 0b00_0_0_001_111_00_1_01000_00000000000101, + }, + .{ + .inst = Instruction.format3a(3, 5, .g0, .o1, .l2), + .expected = 0b11_10010_000101_00000_0_00000000_01001, + }, + .{ + .inst = Instruction.format3b(3, 5, .g0, -1, .l2), + .expected = 0b11_10010_000101_00000_1_1111111111111, + }, + .{ + .inst = Instruction.format3c(3, 5, .g0, .o1), + .expected = 0b11_00000_000101_00000_0_00000000_01001, + }, + .{ + .inst = Instruction.format3d(3, 5, .g0, 0), + .expected = 0b11_00000_000101_00000_1_0000000000000, + }, + .{ + .inst = Instruction.format3e(3, 5, .ne_zero, .g0, .o1, .l2), + .expected = 0b11_10010_000101_00000_0_101_00000_01001, + }, + .{ + .inst = Instruction.format3f(3, 5, .ne_zero, .g0, -1, .l2), + .expected = 0b11_10010_000101_00000_1_101_1111111111, + }, + .{ + .inst = Instruction.format3g(3, 5, .g0, .o1, .l2), + .expected = 0b11_10010_000101_00000_1_00000000_01001, + }, + .{ + .inst = Instruction.format3h(.{}, .{}), + .expected = 0b10_00000_101000_01111_1_000000_000_0000, + }, + .{ + .inst = Instruction.format3i(3, 5, .g0, .o1, .l2, .asi_primary_little), + .expected = 0b11_10010_000101_00000_0_10001000_01001, + }, + .{ + .inst = Instruction.format3j(3, 5, 31, 0), + .expected = 0b11_11111_000101_0000000000000000000, + }, + .{ + .inst = Instruction.format3k(3, 5, .shift32, .g0, .o1, .l2), + .expected = 0b11_10010_000101_00000_0_0_0000000_01001, + }, + .{ + .inst = Instruction.format3l(3, 5, .g0, 31, .l2), + .expected = 0b11_10010_000101_00000_1_0_0000000_11111, + }, + .{ + .inst = Instruction.format3m(3, 5, .g0, 63, .l2), + .expected = 0b11_10010_000101_00000_1_1_000000_111111, + }, + .{ + .inst = Instruction.format3n(3, 5, 0, .o1, .l2), + .expected = 0b11_10010_000101_00000_000000000_01001, + }, + .{ + .inst = Instruction.format3o(3, 5, 0, .xcc, .o1, .l2), + .expected = 0b11_000_1_0_000101_01001_000000000_10010, + }, + .{ + .inst = Instruction.format3p(3, 5, 0, .g0, .o1, .l2), + .expected = 0b11_10010_000101_00000_000000000_01001, + }, + .{ + .inst = Instruction.format3q(3, 5, .g0, .o1), + .expected = 0b11_01001_000101_00000_00000000000000, + }, + .{ + .inst = Instruction.format3r(3, 5, 4), + .expected = 0b11_00100_000101_0000000000000000000, + }, + .{ + .inst = Instruction.format3s(3, 5, .g0), + .expected = 0b11_00000_000101_0000000000000000000, + }, + .{ + .inst = Instruction.format4a(8, .xcc, .g0, .o1, .l2), + .expected = 0b10_10010_001000_00000_0_1_0_000000_01001, + }, + .{ + .inst = Instruction.format4b(8, .xcc, .g0, -1, .l2), + .expected = 0b10_10010_001000_00000_1_1_0_11111111111, + }, + .{ + .inst = Instruction.format4c(8, .{ .icond = .nv }, .xcc, .g0, .o1), + .expected = 0b10_01001_001000_1_0000_0_1_0_000000_00000, + }, + .{ + .inst = Instruction.format4d(8, .{ .icond = .nv }, .xcc, 0, .l2), + .expected = 0b10_10010_001000_1_0000_1_1_0_00000000000, + }, + .{ + .inst = Instruction.format4e(8, .xcc, .g0, .o1, 0), + .expected = 0b10_01001_001000_00000_1_1_0_0000_0000000, + }, + .{ + .inst = Instruction.format4f(8, 4, .eq_zero, .g0, .o1, .l2), + .expected = 0b10_10010_001000_00000_0_001_00100_01001, + }, + .{ + .inst = Instruction.format4g(8, 4, 2, .{ .icond = .nv }, .o1, .l2), + .expected = 0b10_10010_001000_0_0000_010_000100_01001, + }, + }; + + for (testcases) |case| { + const actual = case.inst.toU32(); + testing.expectEqual(case.expected, actual) catch |err| { + std.debug.print("error: {}\n", .{err}); + std.debug.print("case: {}\n", .{case}); + return err; + }; + } +} diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig new file mode 100644 index 0000000000000000000000000000000000000000..d8d8933cc3ea780465a18f3bedf6ed0eefb821d1 --- /dev/null +++ b/src/codegen/wasm/CodeGen.zig @@ -0,0 +1,7578 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const log = std.log.scoped(.codegen); + +const CodeGen = @This(); +const codegen = @import("../../codegen.zig"); +const Zcu = @import("../../Zcu.zig"); +const InternPool = @import("../../InternPool.zig"); +const Decl = Zcu.Decl; +const Type = @import("../../Type.zig"); +const Value = @import("../../Value.zig"); +const Compilation = @import("../../Compilation.zig"); +const link = @import("../../link.zig"); +const Air = @import("../../Air.zig"); +const Mir = @import("Mir.zig"); +const abi = @import("../../codegen/wasm/abi.zig"); +const Alignment = InternPool.Alignment; +const errUnionPayloadOffset = codegen.errUnionPayloadOffset; +const errUnionErrorOffset = codegen.errUnionErrorOffset; + +const target_util = @import("../../target.zig"); +const libcFloatPrefix = target_util.libcFloatPrefix; +const libcFloatSuffix = target_util.libcFloatSuffix; +const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev; +const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev; + +pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { + return comptime &.initMany(&.{ + .expand_intcast_safe, + .expand_int_from_float_safe, + .expand_int_from_float_optimized_safe, + .expand_add_safe, + .expand_sub_safe, + .expand_mul_safe, + }); +} + +/// Reference to the function declaration the code +/// section belongs to +owner_nav: InternPool.Nav.Index, +/// Current block depth. Used to calculate the relative difference between a break +/// and block +block_depth: u32 = 0, +air: Air, +liveness: Air.Liveness, +gpa: mem.Allocator, +func_index: InternPool.Index, +/// Contains a list of current branches. +/// When we return from a branch, the branch will be popped from this list, +/// which means branches can only contain references from within its own branch, +/// or a branch higher (lower index) in the tree. +branches: std.ArrayListUnmanaged(Branch) = .empty, +/// Table to save `WValue`'s generated by an `Air.Inst` +// values: ValueTable, +/// Mapping from Air.Inst.Index to block ids +blocks: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, struct { + label: u32, + value: WValue, +}) = .{}, +/// Maps `loop` instructions to their label. `br` to here repeats the loop. +loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty, +/// The index the next local generated will have +/// NOTE: arguments share the index with locals therefore the first variable +/// will have the index that comes after the last argument's index +local_index: u32, +/// The index of the current argument. +/// Used to track which argument is being referenced in `airArg`. +arg_index: u32 = 0, +/// List of simd128 immediates. Each value is stored as an array of bytes. +/// This list will only be populated for 128bit-simd values when the target features +/// are enabled also. +simd_immediates: std.ArrayListUnmanaged([16]u8) = .empty, +/// The Target we're emitting (used to call intInfo) +target: *const std.Target, +ptr_size: enum { wasm32, wasm64 }, +pt: Zcu.PerThread, +/// List of MIR Instructions +mir_instructions: std.MultiArrayList(Mir.Inst), +/// Contains extra data for MIR +mir_extra: std.ArrayListUnmanaged(u32), +/// List of all locals' types generated throughout this declaration +/// used to emit locals count at start of 'code' section. +mir_locals: std.ArrayListUnmanaged(std.wasm.Valtype), +/// Set of all UAVs referenced by this function. Key is the UAV value, value is the alignment. +/// `.none` means naturally aligned. An explicit alignment is never less than the natural alignment. +mir_uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment), +/// Set of all functions whose address this function has taken and which therefore might be called +/// via a `call_indirect` function. +mir_indirect_function_set: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, void), +/// Set of all function types used by this function. These must be interned by the linker. +mir_func_tys: std.AutoArrayHashMapUnmanaged(InternPool.Index, void), +/// The number of `error_name_table_ref` instructions emitted. +error_name_table_ref_count: u32, +/// When a function is executing, we store the the current stack pointer's value within this local. +/// This value is then used to restore the stack pointer to the original value at the return of the function. +initial_stack_value: WValue = .none, +/// The current stack pointer subtracted with the stack size. From this value, we will calculate +/// all offsets of the stack values. +bottom_stack_value: WValue = .none, +/// Arguments of this function declaration +/// This will be set after `resolveCallingConventionValues` +args: []WValue, +/// This will only be `.none` if the function returns void, or returns an immediate. +/// When it returns a pointer to the stack, the `.local` tag will be active and must be populated +/// before this function returns its execution to the caller. +return_value: WValue, +/// The size of the stack this function occupies. In the function prologue +/// we will move the stack pointer by this number, forward aligned with the `stack_alignment`. +stack_size: u32 = 0, +/// The stack alignment, which is 16 bytes by default. This is specified by the +/// tool-conventions: https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md +/// and also what the llvm backend will emit. +/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default. +stack_alignment: Alignment = .@"16", + +// For each individual Wasm valtype we store a seperate free list which +// allows us to re-use locals that are no longer used. e.g. a temporary local. +/// A list of indexes which represents a local of valtype `i32`. +/// It is illegal to store a non-i32 valtype in this list. +free_locals_i32: std.ArrayListUnmanaged(u32) = .empty, +/// A list of indexes which represents a local of valtype `i64`. +/// It is illegal to store a non-i64 valtype in this list. +free_locals_i64: std.ArrayListUnmanaged(u32) = .empty, +/// A list of indexes which represents a local of valtype `f32`. +/// It is illegal to store a non-f32 valtype in this list. +free_locals_f32: std.ArrayListUnmanaged(u32) = .empty, +/// A list of indexes which represents a local of valtype `f64`. +/// It is illegal to store a non-f64 valtype in this list. +free_locals_f64: std.ArrayListUnmanaged(u32) = .empty, +/// A list of indexes which represents a local of valtype `v127`. +/// It is illegal to store a non-v128 valtype in this list. +free_locals_v128: std.ArrayListUnmanaged(u32) = .empty, + +/// When in debug mode, this tracks if no `finishAir` was missed. +/// Forgetting to call `finishAir` will cause the result to not be +/// stored in our `values` map and therefore cause bugs. +air_bookkeeping: @TypeOf(bookkeeping_init) = bookkeeping_init, + +/// Wasm Value, created when generating an instruction +const WValue = union(enum) { + /// `WValue` which has been freed and may no longer hold + /// any references. + dead: void, + /// May be referenced but is unused + none: void, + /// The value lives on top of the stack + stack: void, + /// Index of the local + local: struct { + /// Contains the index to the local + value: u32, + /// The amount of instructions referencing this `WValue` + references: u32, + }, + /// An immediate 32bit value + imm32: u32, + /// An immediate 64bit value + imm64: u64, + /// Index into the list of simd128 immediates. This `WValue` is + /// only possible in very rare cases, therefore it would be + /// a waste of memory to store the value in a 128 bit integer. + imm128: u32, + /// A constant 32bit float value + float32: f32, + /// A constant 64bit float value + float64: f64, + nav_ref: struct { + nav_index: InternPool.Nav.Index, + offset: i32 = 0, + }, + uav_ref: struct { + ip_index: InternPool.Index, + offset: i32 = 0, + orig_ptr_ty: InternPool.Index = .none, + }, + /// Offset from the bottom of the virtual stack, with the offset + /// pointing to where the value lives. + stack_offset: struct { + /// Contains the actual value of the offset + value: u32, + /// The amount of instructions referencing this `WValue` + references: u32, + }, + + /// Returns the offset from the bottom of the stack. This is useful when + /// we use the load or store instruction to ensure we retrieve the value + /// from the correct position, rather than the value that lives at the + /// bottom of the stack. For instances where `WValue` is not `stack_value` + /// this will return 0, which allows us to simply call this function for all + /// loads and stores without requiring checks everywhere. + fn offset(value: WValue) u32 { + switch (value) { + .stack_offset => |stack_offset| return stack_offset.value, + .dead => unreachable, + else => return 0, + } + } + + /// Promotes a `WValue` to a local when given value is on top of the stack. + /// When encountering a `local` or `stack_offset` this is essentially a no-op. + /// All other tags are illegal. + fn toLocal(value: WValue, gen: *CodeGen, ty: Type) InnerError!WValue { + switch (value) { + .stack => { + const new_local = try gen.allocLocal(ty); + try gen.addLocal(.local_set, new_local.local.value); + return new_local; + }, + .local, .stack_offset => return value, + else => unreachable, + } + } + + /// Marks a local as no longer being referenced and essentially allows + /// us to re-use it somewhere else within the function. + /// The valtype of the local is deducted by using the index of the given `WValue`. + fn free(value: *WValue, gen: *CodeGen) void { + if (value.* != .local) return; + const local_value = value.local.value; + const reserved = gen.args.len + @intFromBool(gen.return_value != .none); + if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals. + + const index = local_value - reserved; + const valtype = gen.mir_locals.items[index]; + switch (valtype) { + .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead + .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return, + .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return, + .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return, + .v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return, + } + log.debug("freed local ({d}) of type {}", .{ local_value, valtype }); + value.* = .dead; + } +}; + +const Op = enum { + @"unreachable", + nop, + block, + loop, + @"if", + @"else", + end, + br, + br_if, + br_table, + @"return", + call, + drop, + select, + global_get, + global_set, + load, + store, + memory_size, + memory_grow, + @"const", + eqz, + eq, + ne, + lt, + gt, + le, + ge, + clz, + ctz, + popcnt, + add, + sub, + mul, + div, + rem, + @"and", + @"or", + xor, + shl, + shr, + rotl, + rotr, + abs, + neg, + ceil, + floor, + trunc, + nearest, + sqrt, + min, + max, + copysign, + wrap, + convert, + demote, + promote, + reinterpret, + extend, +}; + +const OpcodeBuildArguments = struct { + /// First valtype in the opcode (usually represents the type of the output) + valtype1: ?std.wasm.Valtype = null, + /// The operation (e.g. call, unreachable, div, min, sqrt, etc.) + op: Op, + /// Width of the operation (e.g. 8 for i32_load8_s, 16 for i64_extend16_i32_s) + width: ?u8 = null, + /// Second valtype in the opcode name (usually represents the type of the input) + valtype2: ?std.wasm.Valtype = null, + /// Signedness of the op + signedness: ?std.builtin.Signedness = null, +}; + +/// TODO: deprecated, should be split up per tag. +fn buildOpcode(args: OpcodeBuildArguments) std.wasm.Opcode { + switch (args.op) { + .@"unreachable" => unreachable, + .nop => unreachable, + .block => unreachable, + .loop => unreachable, + .@"if" => unreachable, + .@"else" => unreachable, + .end => unreachable, + .br => unreachable, + .br_if => unreachable, + .br_table => unreachable, + .@"return" => unreachable, + .call => unreachable, + .drop => unreachable, + .select => unreachable, + .global_get => unreachable, + .global_set => unreachable, + + .load => if (args.width) |width| switch (width) { + 8 => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_load8_s else return .i32_load8_u, + .i64 => if (args.signedness.? == .signed) return .i64_load8_s else return .i64_load8_u, + .f32, .f64, .v128 => unreachable, + }, + 16 => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_load16_s else return .i32_load16_u, + .i64 => if (args.signedness.? == .signed) return .i64_load16_s else return .i64_load16_u, + .f32, .f64, .v128 => unreachable, + }, + 32 => switch (args.valtype1.?) { + .i64 => if (args.signedness.? == .signed) return .i64_load32_s else return .i64_load32_u, + .i32 => return .i32_load, + .f32 => return .f32_load, + .f64, .v128 => unreachable, + }, + 64 => switch (args.valtype1.?) { + .i64 => return .i64_load, + .f64 => return .f64_load, + else => unreachable, + }, + else => unreachable, + } else switch (args.valtype1.?) { + .i32 => return .i32_load, + .i64 => return .i64_load, + .f32 => return .f32_load, + .f64 => return .f64_load, + .v128 => unreachable, // handled independently + }, + .store => if (args.width) |width| { + switch (width) { + 8 => switch (args.valtype1.?) { + .i32 => return .i32_store8, + .i64 => return .i64_store8, + .f32, .f64, .v128 => unreachable, + }, + 16 => switch (args.valtype1.?) { + .i32 => return .i32_store16, + .i64 => return .i64_store16, + .f32, .f64, .v128 => unreachable, + }, + 32 => switch (args.valtype1.?) { + .i64 => return .i64_store32, + .i32 => return .i32_store, + .f32 => return .f32_store, + .f64, .v128 => unreachable, + }, + 64 => switch (args.valtype1.?) { + .i64 => return .i64_store, + .f64 => return .f64_store, + else => unreachable, + }, + else => unreachable, + } + } else { + switch (args.valtype1.?) { + .i32 => return .i32_store, + .i64 => return .i64_store, + .f32 => return .f32_store, + .f64 => return .f64_store, + .v128 => unreachable, // handled independently + } + }, + + .memory_size => return .memory_size, + .memory_grow => return .memory_grow, + + .@"const" => switch (args.valtype1.?) { + .i32 => return .i32_const, + .i64 => return .i64_const, + .f32 => return .f32_const, + .f64 => return .f64_const, + .v128 => unreachable, // handled independently + }, + + .eqz => switch (args.valtype1.?) { + .i32 => return .i32_eqz, + .i64 => return .i64_eqz, + .f32, .f64, .v128 => unreachable, + }, + .eq => switch (args.valtype1.?) { + .i32 => return .i32_eq, + .i64 => return .i64_eq, + .f32 => return .f32_eq, + .f64 => return .f64_eq, + .v128 => unreachable, // handled independently + }, + .ne => switch (args.valtype1.?) { + .i32 => return .i32_ne, + .i64 => return .i64_ne, + .f32 => return .f32_ne, + .f64 => return .f64_ne, + .v128 => unreachable, // handled independently + }, + + .lt => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_lt_s else return .i32_lt_u, + .i64 => if (args.signedness.? == .signed) return .i64_lt_s else return .i64_lt_u, + .f32 => return .f32_lt, + .f64 => return .f64_lt, + .v128 => unreachable, // handled independently + }, + .gt => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_gt_s else return .i32_gt_u, + .i64 => if (args.signedness.? == .signed) return .i64_gt_s else return .i64_gt_u, + .f32 => return .f32_gt, + .f64 => return .f64_gt, + .v128 => unreachable, // handled independently + }, + .le => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_le_s else return .i32_le_u, + .i64 => if (args.signedness.? == .signed) return .i64_le_s else return .i64_le_u, + .f32 => return .f32_le, + .f64 => return .f64_le, + .v128 => unreachable, // handled independently + }, + .ge => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_ge_s else return .i32_ge_u, + .i64 => if (args.signedness.? == .signed) return .i64_ge_s else return .i64_ge_u, + .f32 => return .f32_ge, + .f64 => return .f64_ge, + .v128 => unreachable, // handled independently + }, + + .clz => switch (args.valtype1.?) { + .i32 => return .i32_clz, + .i64 => return .i64_clz, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .ctz => switch (args.valtype1.?) { + .i32 => return .i32_ctz, + .i64 => return .i64_ctz, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .popcnt => switch (args.valtype1.?) { + .i32 => return .i32_popcnt, + .i64 => return .i64_popcnt, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + + .add => switch (args.valtype1.?) { + .i32 => return .i32_add, + .i64 => return .i64_add, + .f32 => return .f32_add, + .f64 => return .f64_add, + .v128 => unreachable, // handled independently + }, + .sub => switch (args.valtype1.?) { + .i32 => return .i32_sub, + .i64 => return .i64_sub, + .f32 => return .f32_sub, + .f64 => return .f64_sub, + .v128 => unreachable, // handled independently + }, + .mul => switch (args.valtype1.?) { + .i32 => return .i32_mul, + .i64 => return .i64_mul, + .f32 => return .f32_mul, + .f64 => return .f64_mul, + .v128 => unreachable, // handled independently + }, + + .div => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_div_s else return .i32_div_u, + .i64 => if (args.signedness.? == .signed) return .i64_div_s else return .i64_div_u, + .f32 => return .f32_div, + .f64 => return .f64_div, + .v128 => unreachable, // handled independently + }, + .rem => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_rem_s else return .i32_rem_u, + .i64 => if (args.signedness.? == .signed) return .i64_rem_s else return .i64_rem_u, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + + .@"and" => switch (args.valtype1.?) { + .i32 => return .i32_and, + .i64 => return .i64_and, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .@"or" => switch (args.valtype1.?) { + .i32 => return .i32_or, + .i64 => return .i64_or, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .xor => switch (args.valtype1.?) { + .i32 => return .i32_xor, + .i64 => return .i64_xor, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + + .shl => switch (args.valtype1.?) { + .i32 => return .i32_shl, + .i64 => return .i64_shl, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .shr => switch (args.valtype1.?) { + .i32 => if (args.signedness.? == .signed) return .i32_shr_s else return .i32_shr_u, + .i64 => if (args.signedness.? == .signed) return .i64_shr_s else return .i64_shr_u, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .rotl => switch (args.valtype1.?) { + .i32 => return .i32_rotl, + .i64 => return .i64_rotl, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .rotr => switch (args.valtype1.?) { + .i32 => return .i32_rotr, + .i64 => return .i64_rotr, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + + .abs => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_abs, + .f64 => return .f64_abs, + .v128 => unreachable, // handled independently + }, + .neg => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_neg, + .f64 => return .f64_neg, + .v128 => unreachable, // handled independently + }, + .ceil => switch (args.valtype1.?) { + .i64 => unreachable, + .i32 => return .f32_ceil, // when valtype is f16, we store it in i32. + .f32 => return .f32_ceil, + .f64 => return .f64_ceil, + .v128 => unreachable, // handled independently + }, + .floor => switch (args.valtype1.?) { + .i64 => unreachable, + .i32 => return .f32_floor, // when valtype is f16, we store it in i32. + .f32 => return .f32_floor, + .f64 => return .f64_floor, + .v128 => unreachable, // handled independently + }, + .trunc => switch (args.valtype1.?) { + .i32 => if (args.valtype2) |valty| switch (valty) { + .i32 => unreachable, + .i64 => unreachable, + .f32 => if (args.signedness.? == .signed) return .i32_trunc_f32_s else return .i32_trunc_f32_u, + .f64 => if (args.signedness.? == .signed) return .i32_trunc_f64_s else return .i32_trunc_f64_u, + .v128 => unreachable, // handled independently + } else return .f32_trunc, // when no valtype2, it's an f16 instead which is stored in an i32. + .i64 => switch (args.valtype2.?) { + .i32 => unreachable, + .i64 => unreachable, + .f32 => if (args.signedness.? == .signed) return .i64_trunc_f32_s else return .i64_trunc_f32_u, + .f64 => if (args.signedness.? == .signed) return .i64_trunc_f64_s else return .i64_trunc_f64_u, + .v128 => unreachable, // handled independently + }, + .f32 => return .f32_trunc, + .f64 => return .f64_trunc, + .v128 => unreachable, // handled independently + }, + .nearest => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_nearest, + .f64 => return .f64_nearest, + .v128 => unreachable, // handled independently + }, + .sqrt => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_sqrt, + .f64 => return .f64_sqrt, + .v128 => unreachable, // handled independently + }, + .min => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_min, + .f64 => return .f64_min, + .v128 => unreachable, // handled independently + }, + .max => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_max, + .f64 => return .f64_max, + .v128 => unreachable, // handled independently + }, + .copysign => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => return .f32_copysign, + .f64 => return .f64_copysign, + .v128 => unreachable, // handled independently + }, + + .wrap => switch (args.valtype1.?) { + .i32 => switch (args.valtype2.?) { + .i32 => unreachable, + .i64 => return .i32_wrap_i64, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .i64, .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .convert => switch (args.valtype1.?) { + .i32, .i64 => unreachable, + .f32 => switch (args.valtype2.?) { + .i32 => if (args.signedness.? == .signed) return .f32_convert_i32_s else return .f32_convert_i32_u, + .i64 => if (args.signedness.? == .signed) return .f32_convert_i64_s else return .f32_convert_i64_u, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .f64 => switch (args.valtype2.?) { + .i32 => if (args.signedness.? == .signed) return .f64_convert_i32_s else return .f64_convert_i32_u, + .i64 => if (args.signedness.? == .signed) return .f64_convert_i64_s else return .f64_convert_i64_u, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + .v128 => unreachable, // handled independently + }, + .demote => if (args.valtype1.? == .f32 and args.valtype2.? == .f64) return .f32_demote_f64 else unreachable, + .promote => if (args.valtype1.? == .f64 and args.valtype2.? == .f32) return .f64_promote_f32 else unreachable, + .reinterpret => switch (args.valtype1.?) { + .i32 => if (args.valtype2.? == .f32) return .i32_reinterpret_f32 else unreachable, + .i64 => if (args.valtype2.? == .f64) return .i64_reinterpret_f64 else unreachable, + .f32 => if (args.valtype2.? == .i32) return .f32_reinterpret_i32 else unreachable, + .f64 => if (args.valtype2.? == .i64) return .f64_reinterpret_i64 else unreachable, + .v128 => unreachable, // handled independently + }, + .extend => switch (args.valtype1.?) { + .i32 => switch (args.width.?) { + 8 => if (args.signedness.? == .signed) return .i32_extend8_s else unreachable, + 16 => if (args.signedness.? == .signed) return .i32_extend16_s else unreachable, + else => unreachable, + }, + .i64 => switch (args.width.?) { + 8 => if (args.signedness.? == .signed) return .i64_extend8_s else unreachable, + 16 => if (args.signedness.? == .signed) return .i64_extend16_s else unreachable, + 32 => if (args.signedness.? == .signed) return .i64_extend32_s else unreachable, + else => unreachable, + }, + .f32, .f64 => unreachable, + .v128 => unreachable, // handled independently + }, + } +} + +test "Wasm - buildOpcode" { + // Make sure buildOpcode is referenced, and test some examples + const i32_const = buildOpcode(.{ .op = .@"const", .valtype1 = .i32 }); + const i64_extend32_s = buildOpcode(.{ .op = .extend, .valtype1 = .i64, .width = 32, .signedness = .signed }); + const f64_reinterpret_i64 = buildOpcode(.{ .op = .reinterpret, .valtype1 = .f64, .valtype2 = .i64 }); + + try testing.expectEqual(@as(std.wasm.Opcode, .i32_const), i32_const); + try testing.expectEqual(@as(std.wasm.Opcode, .i64_extend32_s), i64_extend32_s); + try testing.expectEqual(@as(std.wasm.Opcode, .f64_reinterpret_i64), f64_reinterpret_i64); +} + +/// Hashmap to store generated `WValue` for each `Air.Inst.Ref` +pub const ValueTable = std.AutoArrayHashMapUnmanaged(Air.Inst.Ref, WValue); + +const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; + +const InnerError = error{ + OutOfMemory, + /// An error occurred when trying to lower AIR to MIR. + CodegenFail, + /// Compiler implementation could not handle a large integer. + Overflow, +} || link.File.UpdateDebugInfoError; + +pub fn deinit(cg: *CodeGen) void { + const gpa = cg.gpa; + for (cg.branches.items) |*branch| branch.deinit(gpa); + cg.branches.deinit(gpa); + cg.blocks.deinit(gpa); + cg.loops.deinit(gpa); + cg.simd_immediates.deinit(gpa); + cg.free_locals_i32.deinit(gpa); + cg.free_locals_i64.deinit(gpa); + cg.free_locals_f32.deinit(gpa); + cg.free_locals_f64.deinit(gpa); + cg.free_locals_v128.deinit(gpa); + cg.mir_instructions.deinit(gpa); + cg.mir_extra.deinit(gpa); + cg.mir_locals.deinit(gpa); + cg.mir_uavs.deinit(gpa); + cg.mir_indirect_function_set.deinit(gpa); + cg.mir_func_tys.deinit(gpa); + cg.* = undefined; +} + +fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { + const zcu = cg.pt.zcu; + const func = zcu.funcInfo(cg.func_index); + return zcu.codegenFail(func.owner_nav, fmt, args); +} + +/// Resolves the `WValue` for the given instruction `inst` +/// When the given instruction has a `Value`, it returns a constant instead +fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue { + var branch_index = cg.branches.items.len; + while (branch_index > 0) : (branch_index -= 1) { + const branch = cg.branches.items[branch_index - 1]; + if (branch.values.get(ref)) |value| { + return value; + } + } + + // when we did not find an existing instruction, it + // means we must generate it from a constant. + // We always store constants in the most outer branch as they must never + // be removed. The most outer branch is always at index 0. + const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref); + assert(!gop.found_existing); + + const pt = cg.pt; + const zcu = pt.zcu; + const val = (try cg.air.value(ref, pt)).?; + const ty = cg.typeOf(ref); + if (!ty.hasRuntimeBitsIgnoreComptime(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) { + gop.value_ptr.* = .none; + return .none; + } + + // When we need to pass the value by reference (such as a struct), we will + // leverage `generateSymbol` to lower the constant to bytes and emit it + // to the 'rodata' section. We then return the index into the section as `WValue`. + // + // In the other cases, we will simply lower the constant to a value that fits + // into a single local (such as a pointer, integer, bool, etc). + const result: WValue = if (isByRef(ty, zcu, cg.target)) + .{ .uav_ref = .{ .ip_index = val.toIntern() } } + else + try cg.lowerConstant(val, ty); + + gop.value_ptr.* = result; + return result; +} + +fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { + const zcu = cg.pt.zcu; + const ty = val.typeOf(zcu); + + return if (isByRef(ty, zcu, cg.target)) + .{ .uav_ref = .{ .ip_index = val.toIntern() } } + else + try cg.lowerConstant(val, ty); +} + +/// NOTE: if result == .stack, it will be stored in .local +fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { + assert(operands.len <= Air.Liveness.bpi - 1); + var tomb_bits = cg.liveness.getTombBits(inst); + for (operands) |operand| { + const dies = @as(u1, @truncate(tomb_bits)) != 0; + tomb_bits >>= 1; + if (!dies) continue; + processDeath(cg, operand); + } + + // results of `none` can never be referenced. + if (result != .none) { + const trackable_result = if (result != .stack) + result + else + try result.toLocal(cg, cg.typeOfIndex(inst)); + const branch = cg.currentBranch(); + branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result); + } + + if (std.debug.runtime_safety) { + cg.air_bookkeeping += 1; + } +} + +const Branch = struct { + values: ValueTable = .{}, + + fn deinit(branch: *Branch, gpa: Allocator) void { + branch.values.deinit(gpa); + branch.* = undefined; + } +}; + +inline fn currentBranch(cg: *CodeGen) *Branch { + return &cg.branches.items[cg.branches.items.len - 1]; +} + +const BigTomb = struct { + gen: *CodeGen, + inst: Air.Inst.Index, + lbt: Air.Liveness.BigTomb, + + fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { + const dies = bt.lbt.feed(); + if (!dies) return; + // This will be a nop for interned constants. + processDeath(bt.gen, op_ref); + } + + fn finishAir(bt: *BigTomb, result: WValue) void { + assert(result != .stack); + if (result != .none) { + bt.gen.currentBranch().values.putAssumeCapacityNoClobber(bt.inst.toRef(), result); + } + + if (std.debug.runtime_safety) { + bt.gen.air_bookkeeping += 1; + } + } +}; + +fn iterateBigTomb(cg: *CodeGen, inst: Air.Inst.Index, operand_count: usize) !BigTomb { + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, operand_count + 1); + return BigTomb{ + .gen = cg, + .inst = inst, + .lbt = cg.liveness.iterateBigTomb(inst), + }; +} + +fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void { + if (ref.toIndex() == null) return; + // Branches are currently only allowed to free locals allocated + // within their own branch. + // TODO: Upon branch consolidation free any locals if needed. + const value = cg.currentBranch().values.getPtr(ref) orelse return; + if (value.* != .local) return; + const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none); + if (value.local.value < reserved_indexes) { + return; // function arguments can never be re-used + } + log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @intFromEnum(ref.toIndex().?), value.local.value }); + value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer + if (value.local.references == 0) { + value.free(cg); + } +} + +fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void { + try cg.mir_instructions.append(cg.gpa, inst); +} + +fn addTag(cg: *CodeGen, tag: Mir.Inst.Tag) error{OutOfMemory}!void { + try cg.addInst(.{ .tag = tag, .data = .{ .tag = {} } }); +} + +fn addExtended(cg: *CodeGen, opcode: std.wasm.MiscOpcode) error{OutOfMemory}!void { + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + try cg.mir_extra.append(cg.gpa, @intFromEnum(opcode)); + try cg.addInst(.{ .tag = .misc_prefix, .data = .{ .payload = extra_index } }); +} + +fn addLabel(cg: *CodeGen, tag: Mir.Inst.Tag, label: u32) error{OutOfMemory}!void { + try cg.addInst(.{ .tag = tag, .data = .{ .label = label } }); +} + +fn addLocal(cg: *CodeGen, tag: Mir.Inst.Tag, local: u32) error{OutOfMemory}!void { + try cg.addInst(.{ .tag = tag, .data = .{ .local = local } }); +} + +/// Accepts an unsigned 32bit integer rather than a signed integer to +/// prevent us from having to bitcast multiple times as most values +/// within codegen are represented as unsigned rather than signed. +fn addImm32(cg: *CodeGen, imm: u32) error{OutOfMemory}!void { + try cg.addInst(.{ .tag = .i32_const, .data = .{ .imm32 = @bitCast(imm) } }); +} + +/// Accepts an unsigned 64bit integer rather than a signed integer to +/// prevent us from having to bitcast multiple times as most values +/// within codegen are represented as unsigned rather than signed. +fn addImm64(cg: *CodeGen, imm: u64) error{OutOfMemory}!void { + const extra_index = try cg.addExtra(Mir.Imm64.init(imm)); + try cg.addInst(.{ .tag = .i64_const, .data = .{ .payload = extra_index } }); +} + +/// Accepts the index into the list of 128bit-immediates +fn addImm128(cg: *CodeGen, index: u32) error{OutOfMemory}!void { + const simd_values = cg.simd_immediates.items[index]; + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + // tag + 128bit value + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 5); + cg.mir_extra.appendAssumeCapacity(@intFromEnum(std.wasm.SimdOpcode.v128_const)); + cg.mir_extra.appendSliceAssumeCapacity(@alignCast(mem.bytesAsSlice(u32, &simd_values))); + try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); +} + +fn addFloat64(cg: *CodeGen, float: f64) error{OutOfMemory}!void { + const extra_index = try cg.addExtra(Mir.Float64.init(float)); + try cg.addInst(.{ .tag = .f64_const, .data = .{ .payload = extra_index } }); +} + +/// Inserts an instruction to load/store from/to wasm's linear memory dependent on the given `tag`. +fn addMemArg(cg: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOfMemory}!void { + const extra_index = try cg.addExtra(mem_arg); + try cg.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } }); +} + +/// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the +/// given `tag`. +fn addAtomicMemArg(cg: *CodeGen, tag: std.wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void { + const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @intFromEnum(tag) })); + _ = try cg.addExtra(mem_arg); + try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); +} + +/// Helper function to emit atomic mir opcodes. +fn addAtomicTag(cg: *CodeGen, tag: std.wasm.AtomicsOpcode) error{OutOfMemory}!void { + const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @intFromEnum(tag) })); + try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); +} + +/// Appends entries to `mir_extra` based on the type of `extra`. +/// Returns the index into `mir_extra` +fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { + const fields = std.meta.fields(@TypeOf(extra)); + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, fields.len); + return cg.addExtraAssumeCapacity(extra); +} + +/// Appends entries to `mir_extra` based on the type of `extra`. +/// Returns the index into `mir_extra` +fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { + const fields = std.meta.fields(@TypeOf(extra)); + const result: u32 = @intCast(cg.mir_extra.items.len); + inline for (fields) |field| { + cg.mir_extra.appendAssumeCapacity(switch (field.type) { + u32 => @field(extra, field.name), + i32 => @bitCast(@field(extra, field.name)), + InternPool.Index, + InternPool.Nav.Index, + => @intFromEnum(@field(extra, field.name)), + else => |field_type| @compileError("Unsupported field type " ++ @typeName(field_type)), + }); + } + return result; +} + +/// For `std.builtin.CallingConvention.auto`. +pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.Valtype { + const ip = &zcu.intern_pool; + return switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(target)) { + 16 => .i32, // stored/loaded as u16 + 32 => .f32, + 64 => .f64, + 80, 128 => .i32, + else => unreachable, + }, + .int, .@"enum" => switch (ty.intInfo(zcu).bits) { + 0...32 => .i32, + 33...64 => .i64, + else => .i32, + }, + .@"struct" => blk: { + if (zcu.typeToPackedStruct(ty)) |packed_struct| { + const backing_int_ty = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); + break :blk typeToValtype(backing_int_ty, zcu, target); + } else { + break :blk .i32; + } + }, + .vector => switch (CodeGen.determineSimdStoreStrategy(ty, zcu, target)) { + .direct => .v128, + .unrolled => .i32, + }, + .@"union" => switch (ty.containerLayout(zcu)) { + .@"packed" => switch (ty.bitSize(zcu)) { + 0...32 => .i32, + 33...64 => .i64, + else => .i32, + }, + else => .i32, + }, + else => .i32, // all represented as reference/immediate + }; +} + +/// Using a given `Type`, returns the corresponding wasm value type +/// Differently from `typeToValtype` this also allows `void` to create a block +/// with no return type +fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType { + return switch (ty.ip_index) { + .void_type, .noreturn_type => .empty, + else => .fromValtype(typeToValtype(ty, zcu, target)), + }; +} + +/// Writes the bytecode depending on the given `WValue` in `val` +fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { + switch (value) { + .dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?) + .none, .stack => {}, // no-op + .local => |idx| try cg.addLocal(.local_get, idx.value), + .imm32 => |val| try cg.addImm32(val), + .imm64 => |val| try cg.addImm64(val), + .imm128 => |val| try cg.addImm128(val), + .float32 => |val| try cg.addInst(.{ .tag = .f32_const, .data = .{ .float32 = val } }), + .float64 => |val| try cg.addFloat64(val), + .nav_ref => |nav_ref| { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + if (ip.getNav(nav_ref.nav_index).isFn(ip)) { + assert(nav_ref.offset == 0); + try cg.mir_indirect_function_set.put(cg.gpa, nav_ref.nav_index, {}); + try cg.addInst(.{ .tag = .func_ref, .data = .{ .nav_index = nav_ref.nav_index } }); + } else if (nav_ref.offset == 0) { + try cg.addInst(.{ .tag = .nav_ref, .data = .{ .nav_index = nav_ref.nav_index } }); + } else { + try cg.addInst(.{ + .tag = .nav_ref_off, + .data = .{ + .payload = try cg.addExtra(Mir.NavRefOff{ + .nav_index = nav_ref.nav_index, + .offset = nav_ref.offset, + }), + }, + }); + } + }, + .uav_ref => |uav| { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + assert(!ip.isFunctionType(ip.typeOf(uav.ip_index))); + const gop = try cg.mir_uavs.getOrPut(cg.gpa, uav.ip_index); + const this_align: Alignment = a: { + if (uav.orig_ptr_ty == .none) break :a .none; + const ptr_type = ip.indexToKey(uav.orig_ptr_ty).ptr_type; + const this_align = ptr_type.flags.alignment; + if (this_align == .none) break :a .none; + const abi_align = Type.fromInterned(ptr_type.child).abiAlignment(zcu); + if (this_align.compare(.lte, abi_align)) break :a .none; + break :a this_align; + }; + if (!gop.found_existing or + gop.value_ptr.* == .none or + (this_align != .none and this_align.compare(.gt, gop.value_ptr.*))) + { + gop.value_ptr.* = this_align; + } + if (uav.offset == 0) { + try cg.addInst(.{ + .tag = .uav_ref, + .data = .{ .ip_index = uav.ip_index }, + }); + } else { + try cg.addInst(.{ + .tag = .uav_ref_off, + .data = .{ .payload = try cg.addExtra(@as(Mir.UavRefOff, .{ + .value = uav.ip_index, + .offset = uav.offset, + })) }, + }); + } + }, + .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset + } +} + +/// If given a local or stack-offset, increases the reference count by 1. +/// The old `WValue` found at instruction `ref` is then replaced by the +/// modified `WValue` and returned. When given a non-local or non-stack-offset, +/// returns the given `operand` itfunc instead. +fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue { + if (operand != .local and operand != .stack_offset) return operand; + var new_value = operand; + switch (new_value) { + .local => |*local| local.references += 1, + .stack_offset => |*stack_offset| stack_offset.references += 1, + else => unreachable, + } + const old_value = cg.getResolvedInst(ref); + old_value.* = new_value; + return new_value; +} + +/// From a reference, returns its resolved `WValue`. +/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet. +fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { + var index = cg.branches.items.len; + while (index > 0) : (index -= 1) { + const branch = cg.branches.items[index - 1]; + if (branch.values.getPtr(ref)) |value| { + return value; + } + } + unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead. +} + +/// Creates one locals for a given `Type`. +/// Returns a corresponding `Wvalue` with `local` as active tag +fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + const valtype = typeToValtype(ty, zcu, cg.target); + const index_or_null = switch (valtype) { + .i32 => cg.free_locals_i32.pop(), + .i64 => cg.free_locals_i64.pop(), + .f32 => cg.free_locals_f32.pop(), + .f64 => cg.free_locals_f64.pop(), + .v128 => cg.free_locals_v128.pop(), + }; + if (index_or_null) |index| { + log.debug("reusing local ({d}) of type {}", .{ index, valtype }); + return .{ .local = .{ .value = index, .references = 1 } }; + } + log.debug("new local of type {}", .{valtype}); + return cg.ensureAllocLocal(ty); +} + +/// Ensures a new local will be created. This is useful when it's useful +/// to use a zero-initialized local. +fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target)); + const initial_index = cg.local_index; + cg.local_index += 1; + return .{ .local = .{ .value = initial_index, .references = 1 } }; +} + +pub const Error = error{ + OutOfMemory, + /// Compiler was asked to operate on a number larger than supported. + Overflow, + /// Indicates the error is already stored in Zcu `failed_codegen`. + CodegenFail, +}; + +pub fn generate( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + air: *const Air, + liveness: *const ?Air.Liveness, +) Error!Mir { + _ = src_loc; + _ = bin_file; + const zcu = pt.zcu; + const gpa = zcu.gpa; + const cg = zcu.funcInfo(func_index); + const file_scope = zcu.navFileScope(cg.owner_nav); + const target = &file_scope.mod.?.resolved_target.result; + const fn_ty = zcu.navValue(cg.owner_nav).typeOf(zcu); + const fn_info = zcu.typeToFunc(fn_ty).?; + const ret_ty: Type = .fromInterned(fn_info.return_type); + const any_returns = !firstParamSRet(fn_info.cc, ret_ty, zcu, target) and ret_ty.hasRuntimeBitsIgnoreComptime(zcu); + + var cc_result = try resolveCallingConventionValues(zcu, fn_ty, target); + defer cc_result.deinit(gpa); + + var code_gen: CodeGen = .{ + .gpa = gpa, + .pt = pt, + .air = air.*, + .liveness = liveness.*.?, + .owner_nav = cg.owner_nav, + .target = target, + .ptr_size = switch (target.cpu.arch) { + .wasm32 => .wasm32, + .wasm64 => .wasm64, + else => unreachable, + }, + .func_index = func_index, + .args = cc_result.args, + .return_value = cc_result.return_value, + .local_index = cc_result.local_index, + .mir_instructions = .empty, + .mir_extra = .empty, + .mir_locals = .empty, + .mir_uavs = .empty, + .mir_indirect_function_set = .empty, + .mir_func_tys = .empty, + .error_name_table_ref_count = 0, + }; + defer code_gen.deinit(); + + try code_gen.mir_func_tys.putNoClobber(gpa, fn_ty.toIntern(), {}); + + return generateInner(&code_gen, any_returns) catch |err| switch (err) { + error.CodegenFail, + error.OutOfMemory, + error.Overflow, + => |e| return e, + else => |e| return code_gen.fail("failed to generate function: {s}", .{@errorName(e)}), + }; +} + +fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir { + const zcu = cg.pt.zcu; + try cg.branches.append(cg.gpa, .{}); + // clean up outer branch + defer { + var outer_branch = cg.branches.pop().?; + outer_branch.deinit(cg.gpa); + assert(cg.branches.items.len == 0); // missing branch merge + } + // Generate MIR for function body + try cg.genBody(cg.air.getMainBody()); + + // In case we have a return value, but the last instruction is a noreturn (such as a while loop) + // we emit an unreachable instruction to tell the stack validator that part will never be reached. + if (any_returns and cg.air.instructions.len > 0) { + const inst: Air.Inst.Index = @enumFromInt(cg.air.instructions.len - 1); + const last_inst_ty = cg.typeOfIndex(inst); + if (!last_inst_ty.hasRuntimeBitsIgnoreComptime(zcu) or last_inst_ty.isNoReturn(zcu)) { + try cg.addTag(.@"unreachable"); + } + } + // End of function body + try cg.addTag(.end); + try cg.addTag(.dbg_epilogue_begin); + + var mir: Mir = .{ + .instructions = cg.mir_instructions.toOwnedSlice(), + .extra = &.{}, // fallible so assigned after errdefer + .locals = &.{}, // fallible so assigned after errdefer + .prologue = if (cg.initial_stack_value == .none) .none else .{ + .sp_local = cg.initial_stack_value.local.value, + .flags = .{ .stack_alignment = cg.stack_alignment }, + .stack_size = cg.stack_size, + .bottom_stack_local = cg.bottom_stack_value.local.value, + }, + .uavs = cg.mir_uavs.move(), + .indirect_function_set = cg.mir_indirect_function_set.move(), + .func_tys = cg.mir_func_tys.move(), + .error_name_table_ref_count = cg.error_name_table_ref_count, + }; + errdefer mir.deinit(cg.gpa); + mir.extra = try cg.mir_extra.toOwnedSlice(cg.gpa); + mir.locals = try cg.mir_locals.toOwnedSlice(cg.gpa); + return mir; +} + +const CallWValues = struct { + args: []WValue, + return_value: WValue, + local_index: u32, + + fn deinit(values: *CallWValues, gpa: Allocator) void { + gpa.free(values.args); + values.* = undefined; + } +}; + +fn resolveCallingConventionValues( + zcu: *const Zcu, + fn_ty: Type, + target: *const std.Target, +) Allocator.Error!CallWValues { + const gpa = zcu.gpa; + const ip = &zcu.intern_pool; + const fn_info = zcu.typeToFunc(fn_ty).?; + const cc = fn_info.cc; + + var result: CallWValues = .{ + .args = &.{}, + .return_value = .none, + .local_index = 0, + }; + if (cc == .naked) return result; + + var args = std.array_list.Managed(WValue).init(gpa); + defer args.deinit(); + + // Check if we store the result as a pointer to the stack rather than + // by value + if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) { + // the sret arg will be passed as first argument, therefore we + // set the `return_value` before allocating locals for regular args. + result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } }; + result.local_index += 1; + } + + switch (cc) { + .auto => { + for (fn_info.param_types.get(ip)) |ty| { + if (!Type.fromInterned(ty).hasRuntimeBitsIgnoreComptime(zcu)) { + continue; + } + + try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + result.local_index += 1; + } + }, + .wasm_mvp => { + for (fn_info.param_types.get(ip)) |ty| { + if (!Type.fromInterned(ty).hasRuntimeBitsIgnoreComptime(zcu)) { + continue; + } + switch (abi.classifyType(.fromInterned(ty), zcu)) { + .direct => |scalar_ty| if (!abi.lowerAsDoubleI64(scalar_ty, zcu)) { + try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + result.local_index += 1; + } else { + try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } }); + result.local_index += 2; + }, + .indirect => { + try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + result.local_index += 1; + }, + } + } + }, + else => unreachable, // Frontend is responsible for emitting an error earlier. + } + result.args = try args.toOwnedSlice(); + return result; +} + +pub fn firstParamSRet( + cc: std.builtin.CallingConvention, + return_type: Type, + zcu: *const Zcu, + target: *const std.Target, +) bool { + if (!return_type.hasRuntimeBitsIgnoreComptime(zcu)) return false; + switch (cc) { + .@"inline" => unreachable, + .auto => return isByRef(return_type, zcu, target), + .wasm_mvp => switch (abi.classifyType(return_type, zcu)) { + .direct => |scalar_ty| return abi.lowerAsDoubleI64(scalar_ty, zcu), + .indirect => return true, + }, + else => return false, + } +} + +/// Lowers a Zig type and its value based on a given calling convention to ensure +/// it matches the ABI. +fn lowerArg(cg: *CodeGen, cc: std.builtin.CallingConvention, ty: Type, value: WValue) !void { + if (cc != .wasm_mvp) { + return cg.lowerToStack(value); + } + + const zcu = cg.pt.zcu; + + switch (abi.classifyType(ty, zcu)) { + .direct => |scalar_type| if (!abi.lowerAsDoubleI64(scalar_type, zcu)) { + if (!isByRef(ty, zcu, cg.target)) { + return cg.lowerToStack(value); + } else { + switch (value) { + .nav_ref, .stack_offset => _ = try cg.load(value, scalar_type, 0), + .dead => unreachable, + else => try cg.emitWValue(value), + } + } + } else { + assert(ty.abiSize(zcu) == 16); + // in this case we have an integer or float that must be lowered as 2 i64's. + try cg.emitWValue(value); + try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 }); + try cg.emitWValue(value); + try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 }); + }, + .indirect => return cg.lowerToStack(value), + } +} + +/// Lowers a `WValue` to the stack. This means when the `value` results in +/// `.stack_offset` we calculate the pointer of this offset and use that. +/// The value is left on the stack, and not stored in any temporary. +fn lowerToStack(cg: *CodeGen, value: WValue) !void { + switch (value) { + .stack_offset => |offset| { + try cg.emitWValue(value); + if (offset.value > 0) { + switch (cg.ptr_size) { + .wasm32 => { + try cg.addImm32(offset.value); + try cg.addTag(.i32_add); + }, + .wasm64 => { + try cg.addImm64(offset.value); + try cg.addTag(.i64_add); + }, + } + } + }, + else => try cg.emitWValue(value), + } +} + +/// Creates a local for the initial stack value +/// Asserts `initial_stack_value` is `.none` +fn initializeStack(cg: *CodeGen) !void { + assert(cg.initial_stack_value == .none); + // Reserve a local to store the current stack pointer + // We can later use this local to set the stack pointer back to the value + // we have stored here. + cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize); + // Also reserve a local to store the bottom stack value + cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize); +} + +/// Reads the stack pointer from `Context.initial_stack_value` and writes it +/// to the global stack pointer variable +fn restoreStackPointer(cg: *CodeGen) !void { + // only restore the pointer if it was initialized + if (cg.initial_stack_value == .none) return; + // Get the original stack pointer's value + try cg.emitWValue(cg.initial_stack_value); + + try cg.addTag(.global_set_sp); +} + +/// From a given type, will create space on the virtual stack to store the value of such type. +/// This returns a `WValue` with its active tag set to `local`, containing the index to the local +/// that points to the position on the virtual stack. This function should be used instead of +/// moveStack unless a local was already created to store the pointer. +/// +/// Asserts Type has codegenbits +fn allocStack(cg: *CodeGen, ty: Type) !WValue { + const pt = cg.pt; + const zcu = pt.zcu; + assert(ty.hasRuntimeBitsIgnoreComptime(zcu)); + if (cg.initial_stack_value == .none) { + try cg.initializeStack(); + } + + const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse { + return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{ + ty.fmt(pt), ty.abiSize(zcu), + }); + }; + const abi_align = ty.abiAlignment(zcu); + + cg.stack_alignment = cg.stack_alignment.max(abi_align); + + const offset: u32 = @intCast(abi_align.forward(cg.stack_size)); + defer cg.stack_size = offset + abi_size; + + return .{ .stack_offset = .{ .value = offset, .references = 1 } }; +} + +/// From a given AIR instruction generates a pointer to the stack where +/// the value of its type will live. +/// This is different from allocStack where this will use the pointer's alignment +/// if it is set, to ensure the stack alignment will be set correctly. +fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue { + const pt = cg.pt; + const zcu = pt.zcu; + const ptr_ty = cg.typeOfIndex(inst); + const pointee_ty = ptr_ty.childType(zcu); + + if (cg.initial_stack_value == .none) { + try cg.initializeStack(); + } + + if (!pointee_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return cg.allocStack(Type.usize); // create a value containing just the stack pointer. + } + + const abi_alignment = ptr_ty.ptrAlignment(zcu); + const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse { + return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{ + pointee_ty.fmt(pt), pointee_ty.abiSize(zcu), + }); + }; + cg.stack_alignment = cg.stack_alignment.max(abi_alignment); + + const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size)); + defer cg.stack_size = offset + abi_size; + + return .{ .stack_offset = .{ .value = offset, .references = 1 } }; +} + +/// From given zig bitsize, returns the wasm bitsize +fn toWasmBits(bits: u16) ?u16 { + return for ([_]u16{ 32, 64, 128 }) |wasm_bits| { + if (bits <= wasm_bits) return wasm_bits; + } else null; +} + +/// Performs a copy of bytes for a given type. Copying all bytes +/// from rhs to lhs. +fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { + const len_known_neq_0 = switch (len) { + .imm32 => |val| if (val != 0) true else return, + .imm64 => |val| if (val != 0) true else return, + else => false, + }; + // When bulk_memory is enabled, we lower it to wasm's memcpy instruction. + // If not, we lower it ourselves manually + if (cg.target.cpu.has(.wasm, .bulk_memory)) { + const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0); + const emit_check = !(len0_ok or len_known_neq_0); + + if (emit_check) { + try cg.startBlock(.block, .empty); + + // Even if `len` is zero, the spec requires an implementation to trap if `src + len` or + // `dst + len` are out of memory bounds. This can easily happen in Zig in a case such + // as: + // + // const dst: [*]u8 = undefined; + // const src: [*]u8 = undefined; + // var len: usize = runtime_zero(); + // @memcpy(dst[0..len], src[0..len]); + // + // So explicitly avoid using `memory.copy` in the `len == 0` case. Lovely design. + try cg.emitWValue(len); + try cg.addTag(.i32_eqz); + try cg.addLabel(.br_if, 0); + } + + try cg.lowerToStack(dst); + try cg.lowerToStack(src); + try cg.emitWValue(len); + try cg.addExtended(.memory_copy); + + if (emit_check) { + try cg.endBlock(); + } + + return; + } + + // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having + // the loop during codegen, rather than inserting a runtime loop into the binary. + switch (len) { + .imm32, .imm64 => blk: { + const length = switch (len) { + .imm32 => |val| val, + .imm64 => |val| val, + else => unreachable, + }; + // if the size (length) is more than 32 bytes, we use a runtime loop instead to prevent + // binary size bloat. + if (length > 32) break :blk; + var offset: u32 = 0; + const lhs_base = dst.offset(); + const rhs_base = src.offset(); + while (offset < length) : (offset += 1) { + // get dst's address to store the result + try cg.emitWValue(dst); + // load byte from src's address + try cg.emitWValue(src); + switch (cg.ptr_size) { + .wasm32 => { + try cg.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 }); + try cg.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 }); + }, + .wasm64 => { + try cg.addMemArg(.i64_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 }); + try cg.addMemArg(.i64_store8, .{ .offset = lhs_base + offset, .alignment = 1 }); + }, + } + } + return; + }, + else => {}, + } + + // allocate a local for the offset, and set it to 0. + // This to ensure that inside loops we correctly re-set the counter. + var offset = try cg.allocLocal(Type.usize); // local for counter + defer offset.free(cg); + switch (cg.ptr_size) { + .wasm32 => try cg.addImm32(0), + .wasm64 => try cg.addImm64(0), + } + try cg.addLocal(.local_set, offset.local.value); + + // outer block to jump to when loop is done + try cg.startBlock(.block, .empty); + try cg.startBlock(.loop, .empty); + + // loop condition (offset == length -> break) + { + try cg.emitWValue(offset); + try cg.emitWValue(len); + switch (cg.ptr_size) { + .wasm32 => try cg.addTag(.i32_eq), + .wasm64 => try cg.addTag(.i64_eq), + } + try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished) + } + + // get dst ptr + { + try cg.emitWValue(dst); + try cg.emitWValue(offset); + switch (cg.ptr_size) { + .wasm32 => try cg.addTag(.i32_add), + .wasm64 => try cg.addTag(.i64_add), + } + } + + // get src value and also store in dst + { + try cg.emitWValue(src); + try cg.emitWValue(offset); + switch (cg.ptr_size) { + .wasm32 => { + try cg.addTag(.i32_add); + try cg.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 }); + try cg.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 }); + }, + .wasm64 => { + try cg.addTag(.i64_add); + try cg.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 }); + try cg.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 }); + }, + } + } + + // increment loop counter + { + try cg.emitWValue(offset); + switch (cg.ptr_size) { + .wasm32 => { + try cg.addImm32(1); + try cg.addTag(.i32_add); + }, + .wasm64 => { + try cg.addImm64(1); + try cg.addTag(.i64_add); + }, + } + try cg.addLocal(.local_set, offset.local.value); + try cg.addLabel(.br, 0); // jump to start of loop + } + try cg.endBlock(); // close off loop block + try cg.endBlock(); // close off outer block +} + +fn ptrSize(cg: *const CodeGen) u16 { + return @divExact(cg.target.ptrBitWidth(), 8); +} + +/// For a given `Type`, will return true when the type will be passed +/// by reference, rather than by value +fn isByRef(ty: Type, zcu: *const Zcu, target: *const std.Target) bool { + const ip = &zcu.intern_pool; + switch (ty.zigTypeTag(zcu)) { + .type, + .comptime_int, + .comptime_float, + .enum_literal, + .undefined, + .null, + .@"opaque", + => unreachable, + + .noreturn, + .void, + .bool, + .error_set, + .@"fn", + .@"anyframe", + => return false, + + .array, + .frame, + => return ty.hasRuntimeBitsIgnoreComptime(zcu), + .@"union" => { + if (zcu.typeToUnion(ty)) |union_obj| { + if (union_obj.flagsUnordered(ip).layout == .@"packed") { + return ty.abiSize(zcu) > 8; + } + } + return ty.hasRuntimeBitsIgnoreComptime(zcu); + }, + .@"struct" => { + if (zcu.typeToPackedStruct(ty)) |packed_struct| { + return isByRef(Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)), zcu, target); + } + return ty.hasRuntimeBitsIgnoreComptime(zcu); + }, + .vector => return determineSimdStoreStrategy(ty, zcu, target) == .unrolled, + .int => return ty.intInfo(zcu).bits > 64, + .@"enum" => return ty.intInfo(zcu).bits > 64, + .float => return ty.floatBits(target) > 64, + .error_union => { + const pl_ty = ty.errorUnionPayload(zcu); + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return false; + } + return true; + }, + .optional => { + if (ty.isPtrLikeOptional(zcu)) return false; + const pl_type = ty.optionalChild(zcu); + if (pl_type.zigTypeTag(zcu) == .error_set) return false; + return pl_type.hasRuntimeBitsIgnoreComptime(zcu); + }, + .pointer => { + // Slices act like struct and will be passed by reference + if (ty.isSlice(zcu)) return true; + return false; + }, + } +} + +const SimdStoreStrategy = enum { + direct, + unrolled, +}; + +/// For a given vector type, returns the `SimdStoreStrategy`. +/// This means when a given type is 128 bits and either the simd128 or relaxed-simd +/// features are enabled, the function will return `.direct`. This would allow to store +/// it using a instruction, rather than an unrolled version. +pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.Target) SimdStoreStrategy { + assert(ty.zigTypeTag(zcu) == .vector); + if (ty.bitSize(zcu) != 128) return .unrolled; + if (target.cpu.has(.wasm, .relaxed_simd) or target.cpu.has(.wasm, .simd128)) { + return .direct; + } + return .unrolled; +} + +/// Creates a new local for a pointer that points to memory with given offset. +/// This can be used to get a pointer to a struct field, error payload, etc. +/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new +/// local value to store the pointer. This allows for local re-use and improves binary size. +fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue { + // do not perform arithmetic when offset is 0. + if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value; + const result_ptr: WValue = switch (action) { + .new => try cg.ensureAllocLocal(Type.usize), + .modify => ptr_value, + }; + try cg.emitWValue(ptr_value); + if (offset + ptr_value.offset() > 0) { + switch (cg.ptr_size) { + .wasm32 => { + try cg.addImm32(@intCast(offset + ptr_value.offset())); + try cg.addTag(.i32_add); + }, + .wasm64 => { + try cg.addImm64(offset + ptr_value.offset()); + try cg.addTag(.i64_add); + }, + } + } + try cg.addLocal(.local_set, result_ptr.local.value); + return result_ptr; +} + +fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const air_tags = cg.air.instructions.items(.tag); + return switch (air_tags[@intFromEnum(inst)]) { + .inferred_alloc, .inferred_alloc_comptime => unreachable, + + .add => cg.airBinOp(inst, .add), + .add_sat => cg.airSatBinOp(inst, .add), + .add_wrap => cg.airWrapBinOp(inst, .add), + .sub => cg.airBinOp(inst, .sub), + .sub_sat => cg.airSatBinOp(inst, .sub), + .sub_wrap => cg.airWrapBinOp(inst, .sub), + .mul => cg.airBinOp(inst, .mul), + .mul_sat => cg.airSatMul(inst), + .mul_wrap => cg.airWrapBinOp(inst, .mul), + .div_float, .div_exact => cg.airDiv(inst), + .div_trunc => cg.airDivTrunc(inst), + .div_floor => cg.airDivFloor(inst), + .bit_and => cg.airBinOp(inst, .@"and"), + .bit_or => cg.airBinOp(inst, .@"or"), + .bool_and => cg.airBinOp(inst, .@"and"), + .bool_or => cg.airBinOp(inst, .@"or"), + .rem => cg.airRem(inst), + .mod => cg.airMod(inst), + .shl => cg.airWrapBinOp(inst, .shl), + .shl_exact => cg.airBinOp(inst, .shl), + .shl_sat => cg.airShlSat(inst), + .shr, .shr_exact => cg.airBinOp(inst, .shr), + .xor => cg.airBinOp(inst, .xor), + .max => cg.airMaxMin(inst, .fmax, .gt), + .min => cg.airMaxMin(inst, .fmin, .lt), + .mul_add => cg.airMulAdd(inst), + + .sqrt => cg.airUnaryFloatOp(inst, .sqrt), + .sin => cg.airUnaryFloatOp(inst, .sin), + .cos => cg.airUnaryFloatOp(inst, .cos), + .tan => cg.airUnaryFloatOp(inst, .tan), + .exp => cg.airUnaryFloatOp(inst, .exp), + .exp2 => cg.airUnaryFloatOp(inst, .exp2), + .log => cg.airUnaryFloatOp(inst, .log), + .log2 => cg.airUnaryFloatOp(inst, .log2), + .log10 => cg.airUnaryFloatOp(inst, .log10), + .floor => cg.airUnaryFloatOp(inst, .floor), + .ceil => cg.airUnaryFloatOp(inst, .ceil), + .round => cg.airUnaryFloatOp(inst, .round), + .trunc_float => cg.airUnaryFloatOp(inst, .trunc), + .neg => cg.airUnaryFloatOp(inst, .neg), + + .abs => cg.airAbs(inst), + + .add_with_overflow => cg.airAddSubWithOverflow(inst, .add), + .sub_with_overflow => cg.airAddSubWithOverflow(inst, .sub), + .shl_with_overflow => cg.airShlWithOverflow(inst), + .mul_with_overflow => cg.airMulWithOverflow(inst), + + .clz => cg.airClz(inst), + .ctz => cg.airCtz(inst), + + .cmp_eq => cg.airCmp(inst, .eq), + .cmp_gte => cg.airCmp(inst, .gte), + .cmp_gt => cg.airCmp(inst, .gt), + .cmp_lte => cg.airCmp(inst, .lte), + .cmp_lt => cg.airCmp(inst, .lt), + .cmp_neq => cg.airCmp(inst, .neq), + + .cmp_vector => cg.airCmpVector(inst), + .cmp_lt_errors_len => cg.airCmpLtErrorsLen(inst), + + .array_elem_val => cg.airArrayElemVal(inst), + .array_to_slice => cg.airArrayToSlice(inst), + .alloc => cg.airAlloc(inst), + .arg => cg.airArg(inst), + .bitcast => cg.airBitcast(inst), + .block => cg.airBlock(inst), + .trap => cg.airTrap(inst), + .breakpoint => cg.airBreakpoint(inst), + .br => cg.airBr(inst), + .repeat => cg.airRepeat(inst), + .switch_dispatch => cg.airSwitchDispatch(inst), + .cond_br => cg.airCondBr(inst), + .intcast => cg.airIntcast(inst), + .fptrunc => cg.airFptrunc(inst), + .fpext => cg.airFpext(inst), + .int_from_float => cg.airIntFromFloat(inst), + .float_from_int => cg.airFloatFromInt(inst), + .get_union_tag => cg.airGetUnionTag(inst), + + .@"try" => cg.airTry(inst), + .try_cold => cg.airTry(inst), + .try_ptr => cg.airTryPtr(inst), + .try_ptr_cold => cg.airTryPtr(inst), + + .dbg_stmt => cg.airDbgStmt(inst), + .dbg_empty_stmt => try cg.finishAir(inst, .none, &.{}), + .dbg_inline_block => cg.airDbgInlineBlock(inst), + .dbg_var_ptr => cg.airDbgVar(inst, .local_var, true), + .dbg_var_val => cg.airDbgVar(inst, .local_var, false), + .dbg_arg_inline => cg.airDbgVar(inst, .arg, false), + + .call => cg.airCall(inst, .auto), + .call_always_tail => cg.airCall(inst, .always_tail), + .call_never_tail => cg.airCall(inst, .never_tail), + .call_never_inline => cg.airCall(inst, .never_inline), + + .is_err => cg.airIsErr(inst, .i32_ne, .value), + .is_non_err => cg.airIsErr(inst, .i32_eq, .value), + .is_err_ptr => cg.airIsErr(inst, .i32_ne, .ptr), + .is_non_err_ptr => cg.airIsErr(inst, .i32_eq, .ptr), + + .is_null => cg.airIsNull(inst, .i32_eq, .value), + .is_non_null => cg.airIsNull(inst, .i32_ne, .value), + .is_null_ptr => cg.airIsNull(inst, .i32_eq, .ptr), + .is_non_null_ptr => cg.airIsNull(inst, .i32_ne, .ptr), + + .load => cg.airLoad(inst), + .loop => cg.airLoop(inst), + .memset => cg.airMemset(inst, false), + .memset_safe => cg.airMemset(inst, true), + .not => cg.airNot(inst), + .optional_payload => cg.airOptionalPayload(inst), + .optional_payload_ptr => cg.airOptionalPayloadPtr(inst), + .optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst), + .ptr_add => cg.airPtrBinOp(inst, .add), + .ptr_sub => cg.airPtrBinOp(inst, .sub), + .ptr_elem_ptr => cg.airPtrElemPtr(inst), + .ptr_elem_val => cg.airPtrElemVal(inst), + .ret => cg.airRet(inst), + .ret_safe => cg.airRet(inst), // TODO + .ret_ptr => cg.airRetPtr(inst), + .ret_load => cg.airRetLoad(inst), + .splat => cg.airSplat(inst), + .select => cg.airSelect(inst), + .shuffle_one => cg.airShuffleOne(inst), + .shuffle_two => cg.airShuffleTwo(inst), + .reduce => cg.airReduce(inst), + .aggregate_init => cg.airAggregateInit(inst), + .union_init => cg.airUnionInit(inst), + .prefetch => cg.airPrefetch(inst), + .popcount => cg.airPopcount(inst), + .byte_swap => cg.airByteSwap(inst), + .bit_reverse => cg.airBitReverse(inst), + + .slice => cg.airSlice(inst), + .slice_len => cg.airSliceLen(inst), + .slice_elem_val => cg.airSliceElemVal(inst), + .slice_elem_ptr => cg.airSliceElemPtr(inst), + .slice_ptr => cg.airSlicePtr(inst), + .ptr_slice_len_ptr => cg.airPtrSliceFieldPtr(inst, cg.ptrSize()), + .ptr_slice_ptr_ptr => cg.airPtrSliceFieldPtr(inst, 0), + .store => cg.airStore(inst, false), + .store_safe => cg.airStore(inst, true), + + .set_union_tag => cg.airSetUnionTag(inst), + .struct_field_ptr => cg.airStructFieldPtr(inst), + .struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0), + .struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1), + .struct_field_ptr_index_2 => cg.airStructFieldPtrIndex(inst, 2), + .struct_field_ptr_index_3 => cg.airStructFieldPtrIndex(inst, 3), + .struct_field_val => cg.airStructFieldVal(inst), + .field_parent_ptr => cg.airFieldParentPtr(inst), + + .switch_br => cg.airSwitchBr(inst, false), + .loop_switch_br => cg.airSwitchBr(inst, true), + .trunc => cg.airTrunc(inst), + .unreach => cg.airUnreachable(inst), + + .wrap_optional => cg.airWrapOptional(inst), + .unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false), + .unwrap_errunion_payload_ptr => cg.airUnwrapErrUnionPayload(inst, true), + .unwrap_errunion_err => cg.airUnwrapErrUnionError(inst, false), + .unwrap_errunion_err_ptr => cg.airUnwrapErrUnionError(inst, true), + .wrap_errunion_payload => cg.airWrapErrUnionPayload(inst), + .wrap_errunion_err => cg.airWrapErrUnionErr(inst), + .errunion_payload_ptr_set => cg.airErrUnionPayloadPtrSet(inst), + .error_name => cg.airErrorName(inst), + + .wasm_memory_size => cg.airWasmMemorySize(inst), + .wasm_memory_grow => cg.airWasmMemoryGrow(inst), + + .memcpy, .memmove => cg.airMemcpy(inst), + + .ret_addr => cg.airRetAddr(inst), + .tag_name => cg.airTagName(inst), + + .error_set_has_value => cg.airErrorSetHasValue(inst), + .frame_addr => cg.airFrameAddress(inst), + + .runtime_nav_ptr => cg.airRuntimeNavPtr(inst), + + .assembly, + + .err_return_trace, + .set_err_return_trace, + .save_err_return_trace_index, + .is_named_enum_value, + .addrspace_cast, + .vector_store_elem, + .c_va_arg, + .c_va_copy, + .c_va_end, + .c_va_start, + => |tag| return cg.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), + + .atomic_load => cg.airAtomicLoad(inst), + .atomic_store_unordered, + .atomic_store_monotonic, + .atomic_store_release, + .atomic_store_seq_cst, + // in WebAssembly, all atomic instructions are sequentially ordered. + => cg.airAtomicStore(inst), + .atomic_rmw => cg.airAtomicRmw(inst), + .cmpxchg_weak => cg.airCmpxchg(inst), + .cmpxchg_strong => cg.airCmpxchg(inst), + + .add_optimized, + .sub_optimized, + .mul_optimized, + .div_float_optimized, + .div_trunc_optimized, + .div_floor_optimized, + .div_exact_optimized, + .rem_optimized, + .mod_optimized, + .neg_optimized, + .cmp_lt_optimized, + .cmp_lte_optimized, + .cmp_eq_optimized, + .cmp_gte_optimized, + .cmp_gt_optimized, + .cmp_neq_optimized, + .cmp_vector_optimized, + .reduce_optimized, + .int_from_float_optimized, + => return cg.fail("TODO implement optimized float mode", .{}), + + .add_safe, + .sub_safe, + .mul_safe, + .intcast_safe, + .int_from_float_safe, + .int_from_float_optimized_safe, + => return cg.fail("TODO implement safety_checked_instructions", .{}), + + .work_item_id, + .work_group_size, + .work_group_id, + => unreachable, + }; +} + +fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + + for (body) |inst| { + if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) { + continue; + } + const old_bookkeeping_value = cg.air_bookkeeping; + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Air.Liveness.bpi); + try cg.genInst(inst); + + if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) { + std.debug.panic("Missing call to `finishAir` in AIR instruction %{d} ('{t}')", .{ + inst, + cg.air.instructions.items(.tag)[@intFromEnum(inst)], + }); + } + } +} + +fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?; + const ret_ty = Type.fromInterned(fn_info.return_type); + + // result must be stored in the stack and we return a pointer + // to the stack instead + if (cg.return_value != .none) { + try cg.store(cg.return_value, operand, ret_ty, 0); + } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + switch (abi.classifyType(ret_ty, zcu)) { + .direct => |scalar_type| { + assert(!abi.lowerAsDoubleI64(scalar_type, zcu)); + if (!isByRef(ret_ty, zcu, cg.target)) { + try cg.emitWValue(operand); + } else { + _ = try cg.load(operand, scalar_type, 0); + } + }, + .indirect => unreachable, + } + } else { + if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and ret_ty.isError(zcu)) { + try cg.addImm32(0); + } else { + try cg.emitWValue(operand); + } + } + try cg.restoreStackPointer(); + try cg.addTag(.@"return"); + + return cg.finishAir(inst, .none, &.{un_op}); +} + +fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const child_type = cg.typeOfIndex(inst).childType(zcu); + + const result = result: { + if (!child_type.isFnOrHasRuntimeBitsIgnoreComptime(zcu)) { + break :result try cg.allocStack(Type.usize); // create pointer to void + } + + const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?; + if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) { + break :result cg.return_value; + } + + break :result try cg.allocStackPtr(inst); + }; + + return cg.finishAir(inst, result, &.{}); +} + +fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + const ret_ty = cg.typeOf(un_op).childType(zcu); + + const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?; + if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + if (ret_ty.isError(zcu)) { + try cg.addImm32(0); + } + } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) { + // leave on the stack + _ = try cg.load(operand, ret_ty, 0); + } + + try cg.restoreStackPointer(); + try cg.addTag(.@"return"); + return cg.finishAir(inst, .none, &.{un_op}); +} + +fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) InnerError!void { + if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{}); + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = cg.air.extraData(Air.Call, pl_op.payload); + const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]); + const ty = cg.typeOf(pl_op.operand); + + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const fn_ty = switch (ty.zigTypeTag(zcu)) { + .@"fn" => ty, + .pointer => ty.childType(zcu), + else => unreachable, + }; + const ret_ty = fn_ty.fnReturnType(zcu); + const fn_info = zcu.typeToFunc(fn_ty).?; + const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target); + + const callee: ?InternPool.Nav.Index = blk: { + const func_val = (try cg.air.value(pl_op.operand, pt)) orelse break :blk null; + + switch (ip.indexToKey(func_val.toIntern())) { + inline .func, .@"extern" => |x| break :blk x.owner_nav, + .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { + .nav => |nav| break :blk nav, + else => {}, + }, + else => {}, + } + return cg.fail("unable to lower callee to a function index", .{}); + }; + + const sret: WValue = if (first_param_sret) blk: { + const sret_local = try cg.allocStack(ret_ty); + try cg.lowerToStack(sret_local); + break :blk sret_local; + } else .none; + + for (args) |arg| { + const arg_val = try cg.resolveInst(arg); + + const arg_ty = cg.typeOf(arg); + if (!arg_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; + + try cg.lowerArg(zcu.typeToFunc(fn_ty).?.cc, arg_ty, arg_val); + } + + if (callee) |nav_index| { + try cg.addInst(.{ .tag = .call_nav, .data = .{ .nav_index = nav_index } }); + } else { + // in this case we call a function pointer + // so load its value onto the stack + assert(ty.zigTypeTag(zcu) == .pointer); + const operand = try cg.resolveInst(pl_op.operand); + try cg.emitWValue(operand); + + try cg.mir_func_tys.put(cg.gpa, fn_ty.toIntern(), {}); + try cg.addInst(.{ + .tag = .call_indirect, + .data = .{ .ip_index = fn_ty.toIntern() }, + }); + } + + const result_value = result_value: { + if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and !ret_ty.isError(zcu)) { + break :result_value .none; + } else if (ret_ty.isNoReturn(zcu)) { + try cg.addTag(.@"unreachable"); + break :result_value .none; + } else if (first_param_sret) { + break :result_value sret; + } else if (zcu.typeToFunc(fn_ty).?.cc == .wasm_mvp) { + switch (abi.classifyType(ret_ty, zcu)) { + .direct => |scalar_type| { + assert(!abi.lowerAsDoubleI64(scalar_type, zcu)); + if (!isByRef(ret_ty, zcu, cg.target)) { + const result_local = try cg.allocLocal(ret_ty); + try cg.addLocal(.local_set, result_local.local.value); + break :result_value result_local; + } else { + const result_local = try cg.allocLocal(ret_ty); + try cg.addLocal(.local_set, result_local.local.value); + const result = try cg.allocStack(ret_ty); + try cg.store(result, result_local, scalar_type, 0); + break :result_value result; + } + }, + .indirect => unreachable, + } + } else { + const result_local = try cg.allocLocal(ret_ty); + try cg.addLocal(.local_set, result_local.local.value); + break :result_value result_local; + } + }; + + var bt = try cg.iterateBigTomb(inst, 1 + args.len); + bt.feed(pl_op.operand); + for (args) |arg| bt.feed(arg); + return bt.finishAir(result_value); +} + +fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const value = try cg.allocStackPtr(inst); + return cg.finishAir(inst, value, &.{}); +} + +fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const ptr_ty = cg.typeOf(bin_op.lhs); + const ptr_info = ptr_ty.ptrInfo(zcu); + const ty = ptr_ty.childType(zcu); + + if (!safety and bin_op.rhs == .undef) { + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); + } + + if (ptr_info.packed_offset.host_size == 0) { + try cg.store(lhs, rhs, ty, 0); + } else { + // at this point we have a non-natural alignment, we must + // load the value, and then shift+or the rhs into the result location. + const host_size = ptr_info.packed_offset.host_size * 8; + const host_ty = try pt.intType(.unsigned, host_size); + const bit_size: u16 = @intCast(ty.bitSize(zcu)); + const bit_offset = ptr_info.packed_offset.bit_offset; + + const mask_val = try cg.resolveValue(val: { + const limbs = try cg.gpa.alloc( + std.math.big.Limb, + std.math.big.int.calcTwosCompLimbCount(host_size) + 1, + ); + defer cg.gpa.free(limbs); + + var mask_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + mask_bigint.setTwosCompIntLimit(.max, .unsigned, host_size); + + if (bit_size != host_size) { + mask_bigint.shiftRight(mask_bigint.toConst(), host_size - bit_size); + } + if (bit_offset != 0) { + mask_bigint.shiftLeft(mask_bigint.toConst(), bit_offset); + } + mask_bigint.bitNotWrap(mask_bigint.toConst(), .unsigned, host_size); + + break :val try pt.intValue_big(host_ty, mask_bigint.toConst()); + }); + + const shift_val: WValue = if (33 <= host_size and host_size <= 64) + .{ .imm64 = bit_offset } + else + .{ .imm32 = bit_offset }; + + if (host_size <= 64) { + try cg.emitWValue(lhs); + } + const loaded = if (host_size <= 64) + try cg.load(lhs, host_ty, 0) + else + lhs; + const anded = try cg.binOp(loaded, mask_val, host_ty, .@"and"); + const extended_value = try cg.intcast(rhs, ty, host_ty); + const shifted_value = if (bit_offset > 0) + try cg.binOp(extended_value, shift_val, host_ty, .shl) + else + extended_value; + const result = try cg.binOp(anded, shifted_value, host_ty, .@"or"); + if (host_size <= 64) { + try cg.store(.stack, result, host_ty, lhs.offset()); + } else { + try cg.store(lhs, result, host_ty, lhs.offset()); + } + } + + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void { + assert(!(lhs != .stack and rhs == .stack)); + const pt = cg.pt; + const zcu = pt.zcu; + const abi_size = ty.abiSize(zcu); + + if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return; + + switch (ty.zigTypeTag(zcu)) { + .error_union => { + const pl_ty = ty.errorUnionPayload(zcu); + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return cg.store(lhs, rhs, Type.anyerror, offset); + } + + const len = @as(u32, @intCast(abi_size)); + assert(offset == 0); + return cg.memcpy(lhs, rhs, .{ .imm32 = len }); + }, + .optional => { + if (ty.isPtrLikeOptional(zcu)) { + return cg.store(lhs, rhs, Type.usize, offset); + } + const pl_ty = ty.optionalChild(zcu); + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return cg.store(lhs, rhs, Type.u8, offset); + } + if (pl_ty.zigTypeTag(zcu) == .error_set) { + return cg.store(lhs, rhs, Type.anyerror, offset); + } + + const len = @as(u32, @intCast(abi_size)); + assert(offset == 0); + return cg.memcpy(lhs, rhs, .{ .imm32 = len }); + }, + .@"struct", .array, .@"union" => if (isByRef(ty, zcu, cg.target)) { + const len = @as(u32, @intCast(abi_size)); + assert(offset == 0); + return cg.memcpy(lhs, rhs, .{ .imm32 = len }); + }, + .vector => switch (determineSimdStoreStrategy(ty, zcu, cg.target)) { + .unrolled => { + const len: u32 = @intCast(abi_size); + return cg.memcpy(lhs, rhs, .{ .imm32 = len }); + }, + .direct => { + try cg.emitWValue(lhs); + try cg.lowerToStack(rhs); + // TODO: Add helper functions for simd opcodes + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + // stores as := opcode, offset, alignment (opcode::memarg) + try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ + @intFromEnum(std.wasm.SimdOpcode.v128_store), + offset + lhs.offset(), + @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0), + }); + return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); + }, + }, + .pointer => { + if (ty.isSlice(zcu)) { + assert(offset == 0); + // store pointer first + // lower it to the stack so we do not have to store rhs into a local first + try cg.emitWValue(lhs); + const ptr_local = try cg.load(rhs, Type.usize, 0); + try cg.store(.stack, ptr_local, Type.usize, 0 + lhs.offset()); + + // retrieve length from rhs, and store that alongside lhs as well + try cg.emitWValue(lhs); + const len_local = try cg.load(rhs, Type.usize, cg.ptrSize()); + try cg.store(.stack, len_local, Type.usize, cg.ptrSize() + lhs.offset()); + return; + } + }, + .int, .@"enum", .float => if (abi_size > 8 and abi_size <= 16) { + assert(offset == 0); + try cg.emitWValue(lhs); + const lsb = try cg.load(rhs, Type.u64, 0); + try cg.store(.stack, lsb, Type.u64, 0 + lhs.offset()); + + try cg.emitWValue(lhs); + const msb = try cg.load(rhs, Type.u64, 8); + try cg.store(.stack, msb, Type.u64, 8 + lhs.offset()); + return; + } else if (abi_size > 16) { + assert(offset == 0); + try cg.memcpy(lhs, rhs, .{ .imm32 = @as(u32, @intCast(ty.abiSize(zcu))) }); + }, + else => if (abi_size > 8) { + return cg.fail("TODO: `store` for type `{f}` with abisize `{d}`", .{ ty.fmt(pt), abi_size }); + }, + } + try cg.emitWValue(lhs); + // In this case we're actually interested in storing the stack position + // into lhs, so we calculate that and emit that instead + try cg.lowerToStack(rhs); + + const valtype = typeToValtype(ty, zcu, cg.target); + const opcode = buildOpcode(.{ + .valtype1 = valtype, + .width = @as(u8, @intCast(abi_size * 8)), + .op = .store, + }); + + // store rhs value at stack pointer's location in memory + try cg.addMemArg( + Mir.Inst.Tag.fromOpcode(opcode), + .{ + .offset = offset + lhs.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }, + ); +} + +fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + const ty = ty_op.ty.toType(); + const ptr_ty = cg.typeOf(ty_op.operand); + const ptr_info = ptr_ty.ptrInfo(zcu); + + if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand}); + + const result = result: { + if (isByRef(ty, zcu, cg.target)) { + const new_local = try cg.allocStack(ty); + try cg.store(new_local, operand, ty, 0); + break :result new_local; + } + + if (ptr_info.packed_offset.host_size == 0) { + const loaded = try cg.load(operand, ty, 0); + const ty_size = ty.abiSize(zcu); + if (ty.isAbiInt(zcu) and ty_size * 8 > ty.bitSize(zcu)) { + const int_elem_ty = try pt.intType(.unsigned, @intCast(ty_size * 8)); + break :result try cg.trunc(loaded, ty, int_elem_ty); + } else { + break :result loaded; + } + } else { + const int_elem_ty = try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8); + const shift_val: WValue = if (ptr_info.packed_offset.host_size <= 4) + .{ .imm32 = ptr_info.packed_offset.bit_offset } + else if (ptr_info.packed_offset.host_size <= 8) + .{ .imm64 = ptr_info.packed_offset.bit_offset } + else + .{ .imm32 = ptr_info.packed_offset.bit_offset }; + + const stack_loaded = if (ptr_info.packed_offset.host_size <= 8) + try cg.load(operand, int_elem_ty, 0) + else + operand; + const shifted = try cg.binOp(stack_loaded, shift_val, int_elem_ty, .shr); + break :result try cg.trunc(shifted, ty, int_elem_ty); + } + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// Loads an operand from the linear memory section. +/// NOTE: Leaves the value on the stack. +fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue { + const zcu = cg.pt.zcu; + // load local's value from memory by its stack position + try cg.emitWValue(operand); + + if (ty.zigTypeTag(zcu) == .vector) { + // TODO: Add helper functions for simd opcodes + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + // stores as := opcode, offset, alignment (opcode::memarg) + try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ + @intFromEnum(std.wasm.SimdOpcode.v128_load), + offset + operand.offset(), + @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }); + try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); + return .stack; + } + + const abi_size: u8 = @intCast(ty.abiSize(zcu)); + const opcode = buildOpcode(.{ + .valtype1 = typeToValtype(ty, zcu, cg.target), + .width = abi_size * 8, + .op = .load, + .signedness = if (ty.isSignedInt(zcu)) .signed else .unsigned, + }); + + try cg.addMemArg( + Mir.Inst.Tag.fromOpcode(opcode), + .{ + .offset = offset + operand.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }, + ); + + return .stack; +} + +fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const arg_index = cg.arg_index; + const arg = cg.args[arg_index]; + const cc = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?.cc; + const arg_ty = cg.typeOfIndex(inst); + if (cc == .wasm_mvp) { + switch (abi.classifyType(arg_ty, zcu)) { + .direct => |scalar_ty| if (!abi.lowerAsDoubleI64(scalar_ty, zcu)) { + cg.arg_index += 1; + } else { + cg.arg_index += 2; + const result = try cg.allocStack(arg_ty); + try cg.store(result, arg, Type.u64, 0); + try cg.store(result, cg.args[arg_index + 1], Type.u64, 8); + return cg.finishAir(inst, result, &.{}); + }, + .indirect => cg.arg_index += 1, + } + } else { + cg.arg_index += 1; + } + + return cg.finishAir(inst, arg, &.{}); +} + +fn airBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const lhs_ty = cg.typeOf(bin_op.lhs); + const rhs_ty = cg.typeOf(bin_op.rhs); + + // For certain operations, such as shifting, the types are different. + // When converting this to a WebAssembly type, they *must* match to perform + // an operation. For this reason we verify if the WebAssembly type is different, in which + // case we first coerce the operands to the same type before performing the operation. + // For big integers we can ignore this as we will call into compiler-rt which handles this. + const result = switch (op) { + .shr, .shl => result: { + if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) { + return cg.fail("TODO: implement vector '{s}' with scalar rhs", .{@tagName(op)}); + } + + const lhs_wasm_bits = toWasmBits(@intCast(lhs_ty.bitSize(zcu))) orelse { + return cg.fail("TODO: implement '{s}' for types larger than 128 bits", .{@tagName(op)}); + }; + const rhs_wasm_bits = toWasmBits(@intCast(rhs_ty.bitSize(zcu))).?; + const new_rhs = if (lhs_wasm_bits != rhs_wasm_bits and lhs_wasm_bits != 128) + try (try cg.intcast(rhs, rhs_ty, lhs_ty)).toLocal(cg, lhs_ty) + else + rhs; + break :result try cg.binOp(lhs, new_rhs, lhs_ty, op); + }, + else => try cg.binOp(lhs, rhs, lhs_ty, op), + }; + + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +/// Performs a binary operation on the given `WValue`'s +/// NOTE: THis leaves the value on top of the stack. +fn binOp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { + const pt = cg.pt; + const zcu = pt.zcu; + assert(!(lhs != .stack and rhs == .stack)); + + if (ty.isAnyFloat()) { + const float_op = FloatOp.fromOp(op); + return cg.floatOp(float_op, ty, &.{ lhs, rhs }); + } + + if (isByRef(ty, zcu, cg.target)) { + if (ty.zigTypeTag(zcu) == .int) { + return cg.binOpBigInt(lhs, rhs, ty, op); + } else { + return cg.fail("TODO: Implement binary operation for type: {f}", .{ty.fmt(pt)}); + } + } + + const opcode: std.wasm.Opcode = buildOpcode(.{ + .op = op, + .valtype1 = typeToValtype(ty, zcu, cg.target), + .signedness = if (ty.isSignedInt(zcu)) .signed else .unsigned, + }); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); + + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + + return .stack; +} + +fn binOpBigInt(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { + const zcu = cg.pt.zcu; + const int_info = ty.intInfo(zcu); + if (int_info.bits > 128) { + return cg.fail("TODO: Implement binary operation for big integers larger than 128 bits", .{}); + } + + switch (op) { + .mul => return cg.callIntrinsic(.__multi3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), + .div => switch (int_info.signedness) { + .signed => return cg.callIntrinsic(.__divti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), + .unsigned => return cg.callIntrinsic(.__udivti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), + }, + .rem => switch (int_info.signedness) { + .signed => return cg.callIntrinsic(.__modti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), + .unsigned => return cg.callIntrinsic(.__umodti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }), + }, + .shr => switch (int_info.signedness) { + .signed => return cg.callIntrinsic(.__ashrti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }), + .unsigned => return cg.callIntrinsic(.__lshrti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }), + }, + .shl => return cg.callIntrinsic(.__ashlti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }), + .@"and", .@"or", .xor => { + const result = try cg.allocStack(ty); + try cg.emitWValue(result); + const lhs_lsb = try cg.load(lhs, Type.u64, 0); + const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const op_lsb = try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, op); + try cg.store(.stack, op_lsb, Type.u64, result.offset()); + + try cg.emitWValue(result); + const lhs_msb = try cg.load(lhs, Type.u64, 8); + const rhs_msb = try cg.load(rhs, Type.u64, 8); + const op_msb = try cg.binOp(lhs_msb, rhs_msb, Type.u64, op); + try cg.store(.stack, op_msb, Type.u64, result.offset() + 8); + return result; + }, + .add, .sub => { + const result = try cg.allocStack(ty); + var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64); + defer lhs_lsb.free(cg); + var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64); + defer rhs_lsb.free(cg); + var op_lsb = try (try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, op)).toLocal(cg, Type.u64); + defer op_lsb.free(cg); + + const lhs_msb = try cg.load(lhs, Type.u64, 8); + const rhs_msb = try cg.load(rhs, Type.u64, 8); + const op_msb = try cg.binOp(lhs_msb, rhs_msb, Type.u64, op); + + const lt = if (op == .add) blk: { + break :blk try cg.cmp(op_lsb, rhs_lsb, Type.u64, .lt); + } else if (op == .sub) blk: { + break :blk try cg.cmp(lhs_lsb, rhs_lsb, Type.u64, .lt); + } else unreachable; + const tmp = try cg.intcast(lt, Type.u32, Type.u64); + var tmp_op = try (try cg.binOp(op_msb, tmp, Type.u64, op)).toLocal(cg, Type.u64); + defer tmp_op.free(cg); + + try cg.store(result, op_lsb, Type.u64, 0); + try cg.store(result, tmp_op, Type.u64, 8); + return result; + }, + else => return cg.fail("TODO: Implement binary operation for big integers: '{s}'", .{@tagName(op)}), + } +} + +const FloatOp = enum { + add, + ceil, + cos, + div, + exp, + exp2, + fabs, + floor, + fma, + fmax, + fmin, + fmod, + log, + log10, + log2, + mul, + neg, + round, + sin, + sqrt, + sub, + tan, + trunc, + + pub fn fromOp(op: Op) FloatOp { + return switch (op) { + .add => .add, + .ceil => .ceil, + .div => .div, + .abs => .fabs, + .floor => .floor, + .max => .fmax, + .min => .fmin, + .mul => .mul, + .neg => .neg, + .nearest => .round, + .sqrt => .sqrt, + .sub => .sub, + .trunc => .trunc, + .rem => .fmod, + else => unreachable, + }; + } + + pub fn toOp(float_op: FloatOp) ?Op { + return switch (float_op) { + .add => .add, + .ceil => .ceil, + .div => .div, + .fabs => .abs, + .floor => .floor, + .fmax => .max, + .fmin => .min, + .mul => .mul, + .neg => .neg, + .round => .nearest, + .sqrt => .sqrt, + .sub => .sub, + .trunc => .trunc, + + .cos, + .exp, + .exp2, + .fma, + .fmod, + .log, + .log10, + .log2, + .sin, + .tan, + => null, + }; + } + + fn intrinsic(op: FloatOp, bits: u16) Mir.Intrinsic { + return switch (op) { + inline .add, .sub, .div, .mul => |ct_op| switch (bits) { + inline 16, 80, 128 => |ct_bits| @field( + Mir.Intrinsic, + "__" ++ @tagName(ct_op) ++ compilerRtFloatAbbrev(ct_bits) ++ "f3", + ), + else => unreachable, + }, + + inline .ceil, + .fabs, + .floor, + .fmax, + .fmin, + .round, + .sqrt, + .trunc, + => |ct_op| switch (bits) { + inline 16, 80, 128 => |ct_bits| @field( + Mir.Intrinsic, + libcFloatPrefix(ct_bits) ++ @tagName(ct_op) ++ libcFloatSuffix(ct_bits), + ), + else => unreachable, + }, + + inline .cos, + .exp, + .exp2, + .fma, + .fmod, + .log, + .log10, + .log2, + .sin, + .tan, + => |ct_op| switch (bits) { + inline 16, 32, 64, 80, 128 => |ct_bits| @field( + Mir.Intrinsic, + libcFloatPrefix(ct_bits) ++ @tagName(ct_op) ++ libcFloatSuffix(ct_bits), + ), + else => unreachable, + }, + + .neg => unreachable, + }; + } +}; + +fn airAbs(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + const ty = cg.typeOf(ty_op.operand); + const scalar_ty = ty.scalarType(zcu); + + switch (scalar_ty.zigTypeTag(zcu)) { + .int => if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)}); + } else { + const int_bits = ty.intInfo(zcu).bits; + const wasm_bits = toWasmBits(int_bits) orelse { + return cg.fail("TODO: airAbs for signed integers larger than '{d}' bits", .{int_bits}); + }; + + switch (wasm_bits) { + 32 => { + try cg.emitWValue(operand); + + try cg.addImm32(31); + try cg.addTag(.i32_shr_s); + + var tmp = try cg.allocLocal(ty); + defer tmp.free(cg); + try cg.addLocal(.local_tee, tmp.local.value); + + try cg.emitWValue(operand); + try cg.addTag(.i32_xor); + try cg.emitWValue(tmp); + try cg.addTag(.i32_sub); + return cg.finishAir(inst, .stack, &.{ty_op.operand}); + }, + 64 => { + try cg.emitWValue(operand); + + try cg.addImm64(63); + try cg.addTag(.i64_shr_s); + + var tmp = try cg.allocLocal(ty); + defer tmp.free(cg); + try cg.addLocal(.local_tee, tmp.local.value); + + try cg.emitWValue(operand); + try cg.addTag(.i64_xor); + try cg.emitWValue(tmp); + try cg.addTag(.i64_sub); + return cg.finishAir(inst, .stack, &.{ty_op.operand}); + }, + 128 => { + const mask = try cg.allocStack(Type.u128); + try cg.emitWValue(mask); + try cg.emitWValue(mask); + + _ = try cg.load(operand, Type.u64, 8); + try cg.addImm64(63); + try cg.addTag(.i64_shr_s); + + var tmp = try cg.allocLocal(Type.u64); + defer tmp.free(cg); + try cg.addLocal(.local_tee, tmp.local.value); + try cg.store(.stack, .stack, Type.u64, mask.offset() + 0); + try cg.emitWValue(tmp); + try cg.store(.stack, .stack, Type.u64, mask.offset() + 8); + + const a = try cg.binOpBigInt(operand, mask, Type.u128, .xor); + const b = try cg.binOpBigInt(a, mask, Type.u128, .sub); + + return cg.finishAir(inst, b, &.{ty_op.operand}); + }, + else => unreachable, + } + }, + .float => { + const result = try cg.floatOp(.fabs, ty, &.{operand}); + return cg.finishAir(inst, result, &.{ty_op.operand}); + }, + else => unreachable, + } +} + +fn airUnaryFloatOp(cg: *CodeGen, inst: Air.Inst.Index, op: FloatOp) InnerError!void { + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + const ty = cg.typeOf(un_op); + + const result = try cg.floatOp(op, ty, &.{operand}); + return cg.finishAir(inst, result, &.{un_op}); +} + +fn floatOp(cg: *CodeGen, float_op: FloatOp, ty: Type, args: []const WValue) InnerError!WValue { + const zcu = cg.pt.zcu; + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: Implement floatOps for vectors", .{}); + } + + const float_bits = ty.floatBits(cg.target); + + if (float_op == .neg) { + return cg.floatNeg(ty, args[0]); + } + + if (float_bits == 32 or float_bits == 64) { + if (float_op.toOp()) |op| { + for (args) |operand| { + try cg.emitWValue(operand); + } + const opcode = buildOpcode(.{ .op = op, .valtype1 = typeToValtype(ty, zcu, cg.target) }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + return .stack; + } + } + + const intrinsic = float_op.intrinsic(float_bits); + + // fma requires three operands + var param_types_buffer: [3]InternPool.Index = .{ ty.ip_index, ty.ip_index, ty.ip_index }; + const param_types = param_types_buffer[0..args.len]; + return cg.callIntrinsic(intrinsic, param_types, ty, args); +} + +/// NOTE: The result value remains on top of the stack. +fn floatNeg(cg: *CodeGen, ty: Type, arg: WValue) InnerError!WValue { + const float_bits = ty.floatBits(cg.target); + switch (float_bits) { + 16 => { + try cg.emitWValue(arg); + try cg.addImm32(0x8000); + try cg.addTag(.i32_xor); + return .stack; + }, + 32, 64 => { + try cg.emitWValue(arg); + const val_type: std.wasm.Valtype = if (float_bits == 32) .f32 else .f64; + const opcode = buildOpcode(.{ .op = .neg, .valtype1 = val_type }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + return .stack; + }, + 80, 128 => { + const result = try cg.allocStack(ty); + try cg.emitWValue(result); + try cg.emitWValue(arg); + try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 }); + + try cg.emitWValue(result); + try cg.emitWValue(arg); + try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 }); + + if (float_bits == 80) { + try cg.addImm64(0x8000); + try cg.addTag(.i64_xor); + try cg.addMemArg(.i64_store16, .{ .offset = 8 + result.offset(), .alignment = 2 }); + } else { + try cg.addImm64(0x8000000000000000); + try cg.addTag(.i64_xor); + try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 }); + } + return result; + }, + else => unreachable, + } +} + +fn airWrapBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const lhs_ty = cg.typeOf(bin_op.lhs); + const rhs_ty = cg.typeOf(bin_op.rhs); + + if (lhs_ty.isVector(zcu)) { + if ((op == .shr or op == .shl) and !rhs_ty.isVector(zcu)) { + return cg.fail("TODO: implement wrapping vector '{s}' with scalar rhs", .{@tagName(op)}); + } else { + return cg.fail("TODO: implement wrapping '{s}' for vectors", .{@tagName(op)}); + } + } + + // For certain operations, such as shifting, the types are different. + // When converting this to a WebAssembly type, they *must* match to perform + // an operation. For this reason we verify if the WebAssembly type is different, in which + // case we first coerce the operands to the same type before performing the operation. + // For big integers we can ignore this as we will call into compiler-rt which handles this. + const result = switch (op) { + .shr, .shl => result: { + const lhs_wasm_bits = toWasmBits(@intCast(lhs_ty.bitSize(zcu))) orelse { + return cg.fail("TODO: implement '{s}' for types larger than 128 bits", .{@tagName(op)}); + }; + const rhs_wasm_bits = toWasmBits(@intCast(rhs_ty.bitSize(zcu))).?; + const new_rhs = if (lhs_wasm_bits != rhs_wasm_bits and lhs_wasm_bits != 128) + try (try cg.intcast(rhs, rhs_ty, lhs_ty)).toLocal(cg, lhs_ty) + else + rhs; + break :result try cg.wrapBinOp(lhs, new_rhs, lhs_ty, op); + }, + else => try cg.wrapBinOp(lhs, rhs, lhs_ty, op), + }; + + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +/// Performs a wrapping binary operation. +/// Asserts rhs is not a stack value when lhs also isn't. +/// NOTE: Leaves the result on the stack when its Type is <= 64 bits +fn wrapBinOp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { + const bin_local = try cg.binOp(lhs, rhs, ty, op); + return cg.wrapOperand(bin_local, ty); +} + +/// Wraps an operand based on a given type's bitsize. +/// Asserts `Type` is <= 128 bits. +/// NOTE: When the Type is <= 64 bits, leaves the value on top of the stack, if wrapping was needed. +fn wrapOperand(cg: *CodeGen, operand: WValue, ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + assert(ty.abiSize(zcu) <= 16); + const int_bits: u16 = @intCast(ty.bitSize(zcu)); // TODO use ty.intInfo(zcu).bits + const wasm_bits = toWasmBits(int_bits) orelse { + return cg.fail("TODO: Implement wrapOperand for bitsize '{d}'", .{int_bits}); + }; + + if (wasm_bits == int_bits) return operand; + + switch (wasm_bits) { + 32 => { + try cg.emitWValue(operand); + if (ty.isSignedInt(zcu)) { + try cg.addImm32(32 - int_bits); + try cg.addTag(.i32_shl); + try cg.addImm32(32 - int_bits); + try cg.addTag(.i32_shr_s); + } else { + try cg.addImm32(~@as(u32, 0) >> @intCast(32 - int_bits)); + try cg.addTag(.i32_and); + } + return .stack; + }, + 64 => { + try cg.emitWValue(operand); + if (ty.isSignedInt(zcu)) { + try cg.addImm64(64 - int_bits); + try cg.addTag(.i64_shl); + try cg.addImm64(64 - int_bits); + try cg.addTag(.i64_shr_s); + } else { + try cg.addImm64(~@as(u64, 0) >> @intCast(64 - int_bits)); + try cg.addTag(.i64_and); + } + return .stack; + }, + 128 => { + assert(operand != .stack); + const result = try cg.allocStack(ty); + + try cg.emitWValue(result); + _ = try cg.load(operand, Type.u64, 0); + try cg.store(.stack, .stack, Type.u64, result.offset()); + + try cg.emitWValue(result); + _ = try cg.load(operand, Type.u64, 8); + if (ty.isSignedInt(zcu)) { + try cg.addImm64(128 - int_bits); + try cg.addTag(.i64_shl); + try cg.addImm64(128 - int_bits); + try cg.addTag(.i64_shr_s); + } else { + try cg.addImm64(~@as(u64, 0) >> @intCast(128 - int_bits)); + try cg.addTag(.i64_and); + } + try cg.store(.stack, .stack, Type.u64, result.offset() + 8); + + return result; + }, + else => unreachable, + } +} + +fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue { + const pt = cg.pt; + const zcu = pt.zcu; + const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr; + const offset: u64 = prev_offset + ptr.byte_offset; + return switch (ptr.base_addr) { + .nav => |nav| return .{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } }, + .uav => |uav| return .{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } }, + .int => return cg.lowerConstant(try pt.intValue(Type.usize, offset), Type.usize), + .eu_payload => |eu_ptr| try cg.lowerPtr( + eu_ptr, + offset + codegen.errUnionPayloadOffset( + Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu), + zcu, + ), + ), + .opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset), + .field => |field| { + const base_ptr = Value.fromInterned(field.base); + const base_ty = base_ptr.typeOf(zcu).childType(zcu); + const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) { + .pointer => off: { + assert(base_ty.isSlice(zcu)); + break :off switch (field.index) { + Value.slice_ptr_index => 0, + Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8), + else => unreachable, + }; + }, + .@"struct" => switch (base_ty.containerLayout(zcu)) { + .auto => base_ty.structFieldOffset(@intCast(field.index), zcu), + .@"extern", .@"packed" => unreachable, + }, + .@"union" => switch (base_ty.containerLayout(zcu)) { + .auto => base_ty.structFieldOffset(@intCast(field.index), zcu), + .@"extern", .@"packed" => unreachable, + }, + else => unreachable, + }; + return cg.lowerPtr(field.base, offset + field_off); + }, + .arr_elem, .comptime_field, .comptime_alloc => unreachable, + }; +} + +/// Asserts that `isByRef` returns `false` for `ty`. +fn lowerConstant(cg: *CodeGen, val: Value, ty: Type) InnerError!WValue { + const pt = cg.pt; + const zcu = pt.zcu; + assert(!isByRef(ty, zcu, cg.target)); + const ip = &zcu.intern_pool; + if (val.isUndef(zcu)) return cg.emitUndefined(ty); + + switch (ip.indexToKey(val.ip_index)) { + .int_type, + .ptr_type, + .array_type, + .vector_type, + .opt_type, + .anyframe_type, + .error_union_type, + .simple_type, + .struct_type, + .tuple_type, + .union_type, + .opaque_type, + .enum_type, + .func_type, + .error_set_type, + .inferred_error_set_type, + => unreachable, // types, not values + + .undef => unreachable, // handled above + .simple_value => |simple_value| switch (simple_value) { + .undefined, + .void, + .null, + .empty_tuple, + .@"unreachable", + => unreachable, // non-runtime values + .false, .true => return .{ .imm32 = switch (simple_value) { + .false => 0, + .true => 1, + else => unreachable, + } }, + }, + .variable, + .@"extern", + .func, + .enum_literal, + .empty_enum_value, + => unreachable, // non-runtime values + .int => { + const int_info = ty.intInfo(zcu); + switch (int_info.signedness) { + .signed => switch (int_info.bits) { + 0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) }, + 33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) }, + else => unreachable, + }, + .unsigned => switch (int_info.bits) { + 0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) }, + 33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) }, + else => unreachable, + }, + } + }, + .err => |err| { + const int = try pt.getErrorValue(err.name); + return .{ .imm32 = int }; + }, + .error_union => |error_union| { + const err_int_ty = try pt.errorIntType(); + const err_ty, const err_val = switch (error_union.val) { + .err_name => |err_name| .{ + ty.errorUnionSet(zcu), + Value.fromInterned(try pt.intern(.{ .err = .{ + .ty = ty.errorUnionSet(zcu).toIntern(), + .name = err_name, + } })), + }, + .payload => .{ + err_int_ty, + try pt.intValue(err_int_ty, 0), + }, + }; + const payload_type = ty.errorUnionPayload(zcu); + if (!payload_type.hasRuntimeBitsIgnoreComptime(zcu)) { + // We use the error type directly as the type. + return cg.lowerConstant(err_val, err_ty); + } + + return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{}); + }, + .enum_tag => |enum_tag| { + const int_tag_ty = ip.typeOf(enum_tag.int); + return cg.lowerConstant(Value.fromInterned(enum_tag.int), Type.fromInterned(int_tag_ty)); + }, + .float => |float| switch (float.storage) { + .f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) }, + .f32 => |f32_val| return .{ .float32 = f32_val }, + .f64 => |f64_val| return .{ .float64 = f64_val }, + else => unreachable, + }, + .slice => unreachable, // isByRef == true + .ptr => return cg.lowerPtr(val.toIntern(), 0), + .opt => if (ty.optionalReprIsPayload(zcu)) { + const pl_ty = ty.optionalChild(zcu); + if (val.optionalValue(zcu)) |payload| { + return cg.lowerConstant(payload, pl_ty); + } else { + return .{ .imm32 = 0 }; + } + } else { + return .{ .imm32 = @intFromBool(!val.isNull(zcu)) }; + }, + .aggregate => switch (ip.indexToKey(ty.ip_index)) { + .array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}), + .vector_type => { + assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct); + var buf: [16]u8 = undefined; + val.writeToMemory(pt, &buf) catch unreachable; + return cg.storeSimdImmd(buf); + }, + .struct_type => { + const struct_type = ip.loadStructType(ty.toIntern()); + // non-packed structs are not handled in this function because they + // are by-ref types. + assert(struct_type.layout == .@"packed"); + var buf: [8]u8 = .{0} ** 8; // zero the buffer so we do not read 0xaa as integer + val.writeToPackedMemory(ty, pt, &buf, 0) catch unreachable; + const backing_int_ty = Type.fromInterned(struct_type.backingIntTypeUnordered(ip)); + const int_val = try pt.intValue( + backing_int_ty, + mem.readInt(u64, &buf, .little), + ); + return cg.lowerConstant(int_val, backing_int_ty); + }, + else => unreachable, + }, + .un => { + const int_type = try pt.intType(.unsigned, @intCast(ty.bitSize(zcu))); + + var buf: [8]u8 = .{0} ** 8; // zero the buffer so we do not read 0xaa as integer + val.writeToPackedMemory(ty, pt, &buf, 0) catch unreachable; + const int_val = try pt.intValue( + int_type, + mem.readInt(u64, &buf, .little), + ); + return cg.lowerConstant(int_val, int_type); + }, + .memoized_call => unreachable, + } +} + +/// Stores the value as a 128bit-immediate value by storing it inside +/// the list and returning the index into this list as `WValue`. +fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue { + const index = @as(u32, @intCast(cg.simd_immediates.items.len)); + try cg.simd_immediates.append(cg.gpa, value); + return .{ .imm128 = index }; +} + +fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + switch (ty.zigTypeTag(zcu)) { + .bool, .error_set => return .{ .imm32 = 0xaaaaaaaa }, + .int, .@"enum" => switch (ty.intInfo(zcu).bits) { + 0...32 => return .{ .imm32 = 0xaaaaaaaa }, + 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, + else => unreachable, + }, + .float => switch (ty.floatBits(cg.target)) { + 16 => return .{ .imm32 = 0xaaaaaaaa }, + 32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) }, + 64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) }, + else => unreachable, + }, + .pointer => switch (cg.ptr_size) { + .wasm32 => return .{ .imm32 = 0xaaaaaaaa }, + .wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, + }, + .optional => { + const pl_ty = ty.optionalChild(zcu); + if (ty.optionalReprIsPayload(zcu)) { + return cg.emitUndefined(pl_ty); + } + return .{ .imm32 = 0xaaaaaaaa }; + }, + .error_union => { + return .{ .imm32 = 0xaaaaaaaa }; + }, + .@"struct" => { + const packed_struct = zcu.typeToPackedStruct(ty).?; + return cg.emitUndefined(Type.fromInterned(packed_struct.backingIntTypeUnordered(ip))); + }, + .@"union" => switch (ty.containerLayout(zcu)) { + .@"packed" => switch (ty.bitSize(zcu)) { + 0...32 => return .{ .imm32 = 0xaaaaaaaa }, + 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, + else => unreachable, + }, + else => unreachable, + }, + else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}), + } +} + +fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.Block, ty_pl.payload); + try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); +} + +fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + // if wasm_block_ty is non-empty, we create a register to store the temporary value + const block_result: WValue = if (block_ty.hasRuntimeBitsIgnoreComptime(zcu)) + try cg.allocLocal(block_ty) + else + .none; + + try cg.startBlock(.block, .empty); + // Here we set the current block idx, so breaks know the depth to jump + // to when breaking out. + try cg.blocks.putNoClobber(cg.gpa, inst, .{ + .label = cg.block_depth, + .value = block_result, + }); + + try cg.genBody(body); + try cg.endBlock(); + + const liveness = cg.liveness.getBlock(inst); + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths.len); + + return cg.finishAir(inst, block_result, &.{}); +} + +/// appends a new wasm block to the code section and increases the `block_depth` by 1 +fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void { + cg.block_depth += 1; + try cg.addInst(.{ + .tag = Mir.Inst.Tag.fromOpcode(block_tag), + .data = .{ .block_type = block_type }, + }); +} + +/// Ends the current wasm block and decreases the `block_depth` by 1 +fn endBlock(cg: *CodeGen) !void { + try cg.addTag(.end); + cg.block_depth -= 1; +} + +fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const loop = cg.air.extraData(Air.Block, ty_pl.payload); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]); + + // result type of loop is always 'noreturn', meaning we can always + // emit the wasm type 'block_empty'. + try cg.startBlock(.loop, .empty); + + try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth); + defer assert(cg.loops.remove(inst)); + + try cg.genBody(body); + try cg.endBlock(); + + return cg.finishAir(inst, .none, &.{}); +} + +fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const condition = try cg.resolveInst(pl_op.operand); + const extra = cg.air.extraData(Air.CondBr, pl_op.payload); + const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const liveness_condbr = cg.liveness.getCondBr(inst); + + // result type is always noreturn, so use `block_empty` as type. + try cg.startBlock(.block, .empty); + // emit the conditional value + try cg.emitWValue(condition); + + // we inserted the block in front of the condition + // so now check if condition matches. If not, break outside this block + // and continue with the then codepath + try cg.addLabel(.br_if, 0); + + try cg.branches.ensureUnusedCapacity(cg.gpa, 2); + { + cg.branches.appendAssumeCapacity(.{}); + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.else_deaths.len))); + defer { + var else_stack = cg.branches.pop().?; + else_stack.deinit(cg.gpa); + } + try cg.genBody(else_body); + try cg.endBlock(); + } + + // Outer block that matches the condition + { + cg.branches.appendAssumeCapacity(.{}); + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.then_deaths.len))); + defer { + var then_stack = cg.branches.pop().?; + then_stack.deinit(cg.gpa); + } + try cg.genBody(then_body); + } + + return cg.finishAir(inst, .none, &.{}); +} + +fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const operand_ty = cg.typeOf(bin_op.lhs); + const result = try cg.cmp(lhs, rhs, operand_ty, op); + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +/// Compares two operands. +/// Asserts rhs is not a stack value when the lhs isn't a stack value either +/// NOTE: This leaves the result on top of the stack, rather than a new local. +fn cmp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOperator) InnerError!WValue { + assert(!(lhs != .stack and rhs == .stack)); + const zcu = cg.pt.zcu; + if (ty.zigTypeTag(zcu) == .optional and !ty.optionalReprIsPayload(zcu)) { + const payload_ty = ty.optionalChild(zcu); + if (payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + // When we hit this case, we must check the value of optionals + // that are not pointers. This means first checking against non-null for + // both lhs and rhs, as well as checking the payload are matching of lhs and rhs + return cg.cmpOptionals(lhs, rhs, ty, op); + } + } else if (ty.isAnyFloat()) { + return cg.cmpFloat(ty, lhs, rhs, op); + } else if (isByRef(ty, zcu, cg.target)) { + return cg.cmpBigInt(lhs, rhs, ty, op); + } + + const signedness: std.builtin.Signedness = blk: { + // by default we tell the operand type is unsigned (i.e. bools and enum values) + if (ty.zigTypeTag(zcu) != .int) break :blk .unsigned; + + // incase of an actual integer, we emit the correct signedness + break :blk ty.intInfo(zcu).signedness; + }; + + // ensure that when we compare pointers, we emit + // the true pointer of a stack value, rather than the stack pointer. + try cg.lowerToStack(lhs); + try cg.lowerToStack(rhs); + + const opcode: std.wasm.Opcode = buildOpcode(.{ + .valtype1 = typeToValtype(ty, zcu, cg.target), + .op = switch (op) { + .lt => .lt, + .lte => .le, + .eq => .eq, + .neq => .ne, + .gte => .ge, + .gt => .gt, + }, + .signedness = signedness, + }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + + return .stack; +} + +/// Compares two floats. +/// NOTE: Leaves the result of the comparison on top of the stack. +fn cmpFloat(cg: *CodeGen, ty: Type, lhs: WValue, rhs: WValue, cmp_op: std.math.CompareOperator) InnerError!WValue { + const float_bits = ty.floatBits(cg.target); + + const op: Op = switch (cmp_op) { + .lt => .lt, + .lte => .le, + .eq => .eq, + .neq => .ne, + .gte => .ge, + .gt => .gt, + }; + + switch (float_bits) { + 16 => { + _ = try cg.fpext(lhs, Type.f16, Type.f32); + _ = try cg.fpext(rhs, Type.f16, Type.f32); + const opcode = buildOpcode(.{ .op = op, .valtype1 = .f32 }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + return .stack; + }, + 32, 64 => { + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); + const val_type: std.wasm.Valtype = if (float_bits == 32) .f32 else .f64; + const opcode = buildOpcode(.{ .op = op, .valtype1 = val_type }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + return .stack; + }, + 80, 128 => { + const intrinsic = floatCmpIntrinsic(cmp_op, float_bits); + const result = try cg.callIntrinsic(intrinsic, &.{ ty.ip_index, ty.ip_index }, Type.bool, &.{ lhs, rhs }); + return cg.cmp(result, .{ .imm32 = 0 }, Type.i32, cmp_op); + }, + else => unreachable, + } +} + +fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + _ = inst; + return cg.fail("TODO implement airCmpVector for wasm", .{}); +} + +fn airCmpLtErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + + try cg.emitWValue(operand); + const pt = cg.pt; + const err_int_ty = try pt.errorIntType(); + try cg.addTag(.errors_len); + const result = try cg.cmp(.stack, .stack, err_int_ty, .lt); + + return cg.finishAir(inst, result, &.{un_op}); +} + +fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br; + const block = cg.blocks.get(br.block_inst).?; + + // if operand has codegen bits we should break with a value + if (block.value != .none) { + const operand = try cg.resolveInst(br.operand); + try cg.lowerToStack(operand); + try cg.addLocal(.local_set, block.value.local.value); + } + + // We map every block to its block index. + // We then determine how far we have to jump to it by subtracting it from current block depth + const idx: u32 = cg.block_depth - block.label; + try cg.addLabel(.br, idx); + + return cg.finishAir(inst, .none, &.{br.operand}); +} + +fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const repeat = cg.air.instructions.items(.data)[@intFromEnum(inst)].repeat; + const loop_label = cg.loops.get(repeat.loop_inst).?; + + const idx: u32 = cg.block_depth - loop_label; + try cg.addLabel(.br, idx); + + return cg.finishAir(inst, .none, &.{}); +} + +fn airNot(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const operand_ty = cg.typeOf(ty_op.operand); + const pt = cg.pt; + const zcu = pt.zcu; + + const result = result: { + if (operand_ty.zigTypeTag(zcu) == .bool) { + try cg.emitWValue(operand); + try cg.addTag(.i32_eqz); + const not_tmp = try cg.allocLocal(operand_ty); + try cg.addLocal(.local_set, not_tmp.local.value); + break :result not_tmp; + } else { + const int_info = operand_ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: Implement binary NOT for {f}", .{operand_ty.fmt(pt)}); + }; + + switch (wasm_bits) { + 32 => { + try cg.emitWValue(operand); + try cg.addImm32(switch (int_info.signedness) { + .unsigned => ~@as(u32, 0) >> @intCast(32 - int_info.bits), + .signed => ~@as(u32, 0), + }); + try cg.addTag(.i32_xor); + break :result .stack; + }, + 64 => { + try cg.emitWValue(operand); + try cg.addImm64(switch (int_info.signedness) { + .unsigned => ~@as(u64, 0) >> @intCast(64 - int_info.bits), + .signed => ~@as(u64, 0), + }); + try cg.addTag(.i64_xor); + break :result .stack; + }, + 128 => { + const ptr = try cg.allocStack(operand_ty); + + try cg.emitWValue(ptr); + _ = try cg.load(operand, Type.u64, 0); + try cg.addImm64(~@as(u64, 0)); + try cg.addTag(.i64_xor); + try cg.store(.stack, .stack, Type.u64, ptr.offset()); + + try cg.emitWValue(ptr); + _ = try cg.load(operand, Type.u64, 8); + try cg.addImm64(switch (int_info.signedness) { + .unsigned => ~@as(u64, 0) >> @intCast(128 - int_info.bits), + .signed => ~@as(u64, 0), + }); + try cg.addTag(.i64_xor); + try cg.store(.stack, .stack, Type.u64, ptr.offset() + 8); + + break :result ptr; + }, + else => unreachable, + } + } + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + try cg.addTag(.@"unreachable"); + return cg.finishAir(inst, .none, &.{}); +} + +fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + // unsupported by wasm itfunc. Can be implemented once we support DWARF + // for wasm + try cg.addTag(.@"unreachable"); + return cg.finishAir(inst, .none, &.{}); +} + +fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + try cg.addTag(.@"unreachable"); + return cg.finishAir(inst, .none, &.{}); +} + +fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + const wanted_ty = cg.typeOfIndex(inst); + const given_ty = cg.typeOf(ty_op.operand); + + const bit_size = given_ty.bitSize(zcu); + const needs_wrapping = (given_ty.isSignedInt(zcu) != wanted_ty.isSignedInt(zcu)) and + bit_size != 32 and bit_size != 64 and bit_size != 128; + + const result = result: { + if (given_ty.isAnyFloat() or wanted_ty.isAnyFloat()) { + break :result try cg.bitcast(wanted_ty, given_ty, operand); + } + + if (isByRef(given_ty, zcu, cg.target) and !isByRef(wanted_ty, zcu, cg.target)) { + const loaded_memory = try cg.load(operand, wanted_ty, 0); + if (needs_wrapping) { + break :result try cg.wrapOperand(loaded_memory, wanted_ty); + } else { + break :result loaded_memory; + } + } + if (!isByRef(given_ty, zcu, cg.target) and isByRef(wanted_ty, zcu, cg.target)) { + const stack_memory = try cg.allocStack(wanted_ty); + try cg.store(stack_memory, operand, given_ty, 0); + if (needs_wrapping) { + break :result try cg.wrapOperand(stack_memory, wanted_ty); + } else { + break :result stack_memory; + } + } + + if (needs_wrapping) { + break :result try cg.wrapOperand(operand, wanted_ty); + } + + break :result switch (operand) { + // for stack offset, return a pointer to this offset. + .stack_offset => try cg.buildPointerOffset(operand, 0, .new), + else => cg.reuseOperand(ty_op.operand, operand), + }; + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn bitcast(cg: *CodeGen, wanted_ty: Type, given_ty: Type, operand: WValue) InnerError!WValue { + const zcu = cg.pt.zcu; + // if we bitcast a float to or from an integer we must use the 'reinterpret' instruction + if (!(wanted_ty.isAnyFloat() or given_ty.isAnyFloat())) return operand; + if (wanted_ty.ip_index == .f16_type or given_ty.ip_index == .f16_type) return operand; + if (wanted_ty.bitSize(zcu) > 64) return operand; + assert((wanted_ty.isInt(zcu) and given_ty.isAnyFloat()) or (wanted_ty.isAnyFloat() and given_ty.isInt(zcu))); + + const opcode = buildOpcode(.{ + .op = .reinterpret, + .valtype1 = typeToValtype(wanted_ty, zcu, cg.target), + .valtype2 = typeToValtype(given_ty, zcu, cg.target), + }); + try cg.emitWValue(operand); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + return .stack; +} + +fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.StructField, ty_pl.payload); + + const struct_ptr = try cg.resolveInst(extra.data.struct_operand); + const struct_ptr_ty = cg.typeOf(extra.data.struct_operand); + const struct_ty = struct_ptr_ty.childType(zcu); + const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index); + return cg.finishAir(inst, result, &.{extra.data.struct_operand}); +} + +fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const struct_ptr = try cg.resolveInst(ty_op.operand); + const struct_ptr_ty = cg.typeOf(ty_op.operand); + const struct_ty = struct_ptr_ty.childType(zcu); + + const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn structFieldPtr( + cg: *CodeGen, + inst: Air.Inst.Index, + ref: Air.Inst.Ref, + struct_ptr: WValue, + struct_ptr_ty: Type, + struct_ty: Type, + index: u32, +) InnerError!WValue { + const pt = cg.pt; + const zcu = pt.zcu; + const result_ty = cg.typeOfIndex(inst); + const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu); + + const offset = switch (struct_ty.containerLayout(zcu)) { + .@"packed" => switch (struct_ty.zigTypeTag(zcu)) { + .@"struct" => offset: { + if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) { + break :offset @as(u32, 0); + } + const struct_type = zcu.typeToStruct(struct_ty).?; + break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8); + }, + .@"union" => 0, + else => unreachable, + }, + else => struct_ty.structFieldOffset(index, zcu), + }; + // save a load and store when we can simply reuse the operand + if (offset == 0) { + return cg.reuseOperand(ref, struct_ptr); + } + switch (struct_ptr) { + .stack_offset => |stack_offset| { + return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } }; + }, + else => return cg.buildPointerOffset(struct_ptr, offset, .new), + } +} + +fn airStructFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; + + const struct_ty = cg.typeOf(struct_field.struct_operand); + const operand = try cg.resolveInst(struct_field.struct_operand); + const field_index = struct_field.field_index; + const field_ty = struct_ty.fieldType(field_index, zcu); + if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand}); + + const result: WValue = switch (struct_ty.containerLayout(zcu)) { + .@"packed" => switch (struct_ty.zigTypeTag(zcu)) { + .@"struct" => result: { + const packed_struct = zcu.typeToPackedStruct(struct_ty).?; + const offset = zcu.structPackedFieldBitOffset(packed_struct, field_index); + const backing_ty = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); + const host_bits = backing_ty.intInfo(zcu).bits; + + const const_wvalue: WValue = if (33 <= host_bits and host_bits <= 64) + .{ .imm64 = offset } + else + .{ .imm32 = offset }; + + // for first field we don't require any shifting + const shifted_value = if (offset == 0) + operand + else + try cg.binOp(operand, const_wvalue, backing_ty, .shr); + + if (field_ty.zigTypeTag(zcu) == .float) { + const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); + const truncated = try cg.trunc(shifted_value, int_type, backing_ty); + break :result try cg.bitcast(field_ty, int_type, truncated); + } else if (field_ty.isPtrAtRuntime(zcu) and packed_struct.field_types.len == 1) { + // In this case we do not have to perform any transformations, + // we can simply reuse the operand. + break :result cg.reuseOperand(struct_field.struct_operand, operand); + } else if (field_ty.isPtrAtRuntime(zcu)) { + const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); + break :result try cg.trunc(shifted_value, int_type, backing_ty); + } + break :result try cg.trunc(shifted_value, field_ty, backing_ty); + }, + .@"union" => result: { + if (isByRef(struct_ty, zcu, cg.target)) { + if (!isByRef(field_ty, zcu, cg.target)) { + break :result try cg.load(operand, field_ty, 0); + } else { + const new_stack_val = try cg.allocStack(field_ty); + try cg.store(new_stack_val, operand, field_ty, 0); + break :result new_stack_val; + } + } + + const union_int_type = try pt.intType(.unsigned, @as(u16, @intCast(struct_ty.bitSize(zcu)))); + if (field_ty.zigTypeTag(zcu) == .float) { + const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); + const truncated = try cg.trunc(operand, int_type, union_int_type); + break :result try cg.bitcast(field_ty, int_type, truncated); + } else if (field_ty.isPtrAtRuntime(zcu)) { + const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu)))); + break :result try cg.trunc(operand, int_type, union_int_type); + } + break :result try cg.trunc(operand, field_ty, union_int_type); + }, + else => unreachable, + }, + else => result: { + const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse { + return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)}); + }; + if (isByRef(field_ty, zcu, cg.target)) { + switch (operand) { + .stack_offset => |stack_offset| { + break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } }; + }, + else => break :result try cg.buildPointerOffset(operand, offset, .new), + } + } + break :result try cg.load(operand, field_ty, offset); + }, + }; + + return cg.finishAir(inst, result, &.{struct_field.struct_operand}); +} + +fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + + const switch_br = cg.air.unwrapSwitch(inst); + const target_ty = cg.typeOf(switch_br.operand); + + assert(target_ty.hasRuntimeBitsIgnoreComptime(zcu)); + + // swap target value with placeholder local, for dispatching + const target = if (is_dispatch_loop) target: { + const initial_target = try cg.resolveInst(switch_br.operand); + const target: WValue = try cg.allocLocal(target_ty); + try cg.lowerToStack(initial_target); + try cg.addLocal(.local_set, target.local.value); + + try cg.startBlock(.loop, .empty); // dispatch loop start + try cg.blocks.putNoClobber(cg.gpa, inst, .{ + .label = cg.block_depth, + .value = target, + }); + + break :target target; + } else try cg.resolveInst(switch_br.operand); + + const liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1); + defer cg.gpa.free(liveness.deaths); + + const has_else_body = switch_br.else_body_len != 0; + const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions + try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body)); + + if (switch_br.cases_len == 0) { + assert(has_else_body); + + var it = switch_br.iterateCases(); + const else_body = it.elseBody(); + + cg.branches.appendAssumeCapacity(.{}); + const else_deaths = liveness.deaths.len - 1; + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len); + defer { + var else_branch = cg.branches.pop().?; + else_branch.deinit(cg.gpa); + } + try cg.genBody(else_body); + + if (is_dispatch_loop) { + try cg.endBlock(); // dispatch loop end + } + return cg.finishAir(inst, .none, &.{}); + } + + var min: ?Value = null; + var max: ?Value = null; + var branching_size: u32 = 0; // single item +1, range +2 + + { + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val; + if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val; + branching_size += 1; + } + for (case.ranges) |range| { + const low = Value.fromInterned(range[0].toInterned().?); + if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low; + const high = Value.fromInterned(range[1].toInterned().?); + if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high; + branching_size += 2; + } + } + } + + var min_space: Value.BigIntSpace = undefined; + const min_bigint = min.?.toBigInt(&min_space, zcu); + var max_space: Value.BigIntSpace = undefined; + const max_bigint = max.?.toBigInt(&max_space, zcu); + const limbs = try cg.gpa.alloc( + std.math.big.Limb, + @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1, + ); + defer cg.gpa.free(limbs); + + const width_maybe: ?u32 = width: { + var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + width_bigint.sub(max_bigint, min_bigint); + width_bigint.addScalar(width_bigint.toConst(), 1); + break :width width_bigint.toConst().toInt(u32) catch null; + }; + + try cg.startBlock(.block, .empty); // whole switch block start + + for (0..branch_count) |_| { + try cg.startBlock(.block, .empty); + } + + // Heuristic on deciding when to use .br_table instead of .br_if jump table + // 1. Differences between lowest and highest values should fit into u32 + // 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions + // 3. Do not use .br_table for tiny switches + const use_br_table = cond: { + const width = width_maybe orelse break :cond false; + if (width > 2 * branching_size) break :cond false; + if (width < 2 or branch_count < 2) break :cond false; + break :cond true; + }; + + if (use_br_table) { + const width = width_maybe.?; + + const br_value_original = try cg.binOp(target, try cg.resolveValue(min.?), target_ty, .sub); + _ = try cg.intcast(br_value_original, target_ty, Type.u32); + + const jump_table: Mir.JumpTable = .{ .length = width + 1 }; + const table_extra_index = try cg.addExtra(jump_table); + try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } }); + + const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1); + @memset(branch_list, branch_count - 1); + + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + var val_space: Value.BigIntSpace = undefined; + const val_bigint = val.toBigInt(&val_space, zcu); + var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + index_bigint.sub(val_bigint, min_bigint); + branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx; + } + for (case.ranges) |range| { + var low_space: Value.BigIntSpace = undefined; + const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu); + var high_space: Value.BigIntSpace = undefined; + const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu); + var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + index_bigint.sub(low_bigint, min_bigint); + const start = index_bigint.toConst().toInt(u32) catch unreachable; + index_bigint.sub(high_bigint, min_bigint); + const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1; + @memset(branch_list[start..end], case.idx); + } + } + } else { + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + for (case.items) |ref| { + const val = try cg.resolveInst(ref); + _ = try cg.cmp(target, val, target_ty, .eq); + try cg.addLabel(.br_if, case.idx); // item match found + } + for (case.ranges) |range| { + const low = try cg.resolveInst(range[0]); + const high = try cg.resolveInst(range[1]); + + const gte = try cg.cmp(target, low, target_ty, .gte); + const lte = try cg.cmp(target, high, target_ty, .lte); + _ = try cg.binOp(gte, lte, Type.bool, .@"and"); + try cg.addLabel(.br_if, case.idx); // range match found + } + } + try cg.addLabel(.br, branch_count - 1); + } + + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + try cg.endBlock(); + + cg.branches.appendAssumeCapacity(.{}); + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[case.idx].len); + defer { + var case_branch = cg.branches.pop().?; + case_branch.deinit(cg.gpa); + } + try cg.genBody(case.body); + + try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch + } + + try cg.endBlock(); + if (has_else_body) { + const else_body = cases_it.elseBody(); + + cg.branches.appendAssumeCapacity(.{}); + const else_deaths = liveness.deaths.len - 1; + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len); + defer { + var else_branch = cg.branches.pop().?; + else_branch.deinit(cg.gpa); + } + try cg.genBody(else_body); + } else { + try cg.addTag(.@"unreachable"); + } + + try cg.endBlock(); // whole switch block end + + if (is_dispatch_loop) { + try cg.endBlock(); // dispatch loop end + } + + return cg.finishAir(inst, .none, &.{}); +} + +fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br; + const switch_loop = cg.blocks.get(br.block_inst).?; + + const operand = try cg.resolveInst(br.operand); + try cg.lowerToStack(operand); + try cg.addLocal(.local_set, switch_loop.value.local.value); + + const idx: u32 = cg.block_depth - switch_loop.label; + try cg.addLabel(.br, idx); + + return cg.finishAir(inst, .none, &.{br.operand}); +} + +fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void { + const zcu = cg.pt.zcu; + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + const err_union_ty = cg.typeOf(un_op); + const pl_ty = err_union_ty.errorUnionPayload(zcu); + + const result: WValue = result: { + if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { + switch (opcode) { + .i32_ne => break :result .{ .imm32 = 0 }, + .i32_eq => break :result .{ .imm32 = 1 }, + else => unreachable, + } + } + + try cg.emitWValue(operand); + if (op_kind == .ptr or pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + try cg.addMemArg(.i32_load16_u, .{ + .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))), + .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?), + }); + } + + // Compare the error value with '0' + try cg.addImm32(0); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + break :result .stack; + }; + return cg.finishAir(inst, result, &.{un_op}); +} + +/// E!T -> T op_is_ptr == false +/// *(E!T) -> *T op_is_prt == true +fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const op_ty = cg.typeOf(ty_op.operand); + const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; + const payload_ty = eu_ty.errorUnionPayload(zcu); + + const result: WValue = result: { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + if (op_is_ptr) { + break :result cg.reuseOperand(ty_op.operand, operand); + } else { + break :result .none; + } + } + + const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); + if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) { + break :result try cg.buildPointerOffset(operand, pl_offset, .new); + } else { + assert(isByRef(eu_ty, zcu, cg.target)); + break :result try cg.load(operand, payload_ty, pl_offset); + } + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// E!T -> E op_is_ptr == false +/// *(E!T) -> E op_is_prt == true +/// NOTE: op_is_ptr will not change return type +fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const op_ty = cg.typeOf(ty_op.operand); + const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; + const payload_ty = eu_ty.errorUnionPayload(zcu); + + const result: WValue = result: { + if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { + break :result .{ .imm32 = 0 }; + } + + const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu)); + if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) { + break :result try cg.load(operand, Type.anyerror, err_offset); + } else { + assert(!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)); + break :result cg.reuseOperand(ty_op.operand, operand); + } + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const err_ty = cg.typeOfIndex(inst); + + const pl_ty = cg.typeOf(ty_op.operand); + const result = result: { + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :result cg.reuseOperand(ty_op.operand, operand); + } + + const err_union = try cg.allocStack(err_ty); + const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); + try cg.store(payload_ptr, operand, pl_ty, 0); + + // ensure we also write '0' to the error part, so any present stack value gets overwritten by it. + try cg.emitWValue(err_union); + try cg.addImm32(0); + const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + try cg.addMemArg(.i32_store16, .{ + .offset = err_union.offset() + err_val_offset, + .alignment = 2, + }); + break :result err_union; + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const err_ty = ty_op.ty.toType(); + const pl_ty = err_ty.errorUnionPayload(zcu); + + const result = result: { + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :result cg.reuseOperand(ty_op.operand, operand); + } + + const err_union = try cg.allocStack(err_ty); + // store error value + try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu))); + + // write 'undefined' to the payload + const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); + const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu))); + try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa }); + + break :result err_union; + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airIntcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const ty = ty_op.ty.toType(); + const operand = try cg.resolveInst(ty_op.operand); + const operand_ty = cg.typeOf(ty_op.operand); + const zcu = cg.pt.zcu; + if (ty.zigTypeTag(zcu) == .vector or operand_ty.zigTypeTag(zcu) == .vector) { + return cg.fail("todo Wasm intcast for vectors", .{}); + } + if (ty.abiSize(zcu) > 16 or operand_ty.abiSize(zcu) > 16) { + return cg.fail("todo Wasm intcast for bitsize > 128", .{}); + } + + const op_bits = toWasmBits(@intCast(operand_ty.bitSize(zcu))).?; + const wanted_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?; + const result = if (op_bits == wanted_bits) + cg.reuseOperand(ty_op.operand, operand) + else + try cg.intcast(operand, operand_ty, ty); + + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// Upcasts or downcasts an integer based on the given and wanted types, +/// and stores the result in a new operand. +/// Asserts type's bitsize <= 128 +/// NOTE: May leave the result on the top of the stack. +fn intcast(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + const given_bitsize = @as(u16, @intCast(given.bitSize(zcu))); + const wanted_bitsize = @as(u16, @intCast(wanted.bitSize(zcu))); + assert(given_bitsize <= 128); + assert(wanted_bitsize <= 128); + + const op_bits = toWasmBits(given_bitsize).?; + const wanted_bits = toWasmBits(wanted_bitsize).?; + if (op_bits == wanted_bits) { + return operand; + } + + if (op_bits == 64 and wanted_bits == 32) { + try cg.emitWValue(operand); + try cg.addTag(.i32_wrap_i64); + return .stack; + } else if (op_bits == 32 and wanted_bits == 64) { + try cg.emitWValue(operand); + try cg.addTag(if (wanted.isSignedInt(zcu)) .i64_extend_i32_s else .i64_extend_i32_u); + return .stack; + } else if (wanted_bits == 128) { + // for 128bit integers we store the integer in the virtual stack, rather than a local + const stack_ptr = try cg.allocStack(wanted); + try cg.emitWValue(stack_ptr); + + // for 32 bit integers, we first coerce the value into a 64 bit integer before storing it + // meaning less store operations are required. + const lhs = if (op_bits == 32) blk: { + const sign_ty = if (wanted.isSignedInt(zcu)) Type.i64 else Type.u64; + break :blk try (try cg.intcast(operand, given, sign_ty)).toLocal(cg, sign_ty); + } else operand; + + // store lsb first + try cg.store(.stack, lhs, Type.u64, 0 + stack_ptr.offset()); + + // For signed integers we shift lsb by 63 (64bit integer - 1 sign bit) and store remaining value + if (wanted.isSignedInt(zcu)) { + try cg.emitWValue(stack_ptr); + const shr = try cg.binOp(lhs, .{ .imm64 = 63 }, Type.i64, .shr); + try cg.store(.stack, shr, Type.u64, 8 + stack_ptr.offset()); + } else { + // Ensure memory of msb is zero'd + try cg.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8); + } + return stack_ptr; + } else return cg.load(operand, wanted, 0); +} + +fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void { + const zcu = cg.pt.zcu; + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + + const op_ty = cg.typeOf(un_op); + const optional_ty = if (op_kind == .ptr) op_ty.childType(zcu) else op_ty; + const result = try cg.isNull(operand, optional_ty, opcode); + return cg.finishAir(inst, result, &.{un_op}); +} + +/// For a given type and operand, checks if it's considered `null`. +/// NOTE: Leaves the result on the stack +fn isNull(cg: *CodeGen, operand: WValue, optional_ty: Type, opcode: std.wasm.Opcode) InnerError!WValue { + const pt = cg.pt; + const zcu = pt.zcu; + try cg.emitWValue(operand); + const payload_ty = optional_ty.optionalChild(zcu); + if (!optional_ty.optionalReprIsPayload(zcu)) { + // When payload is zero-bits, we can treat operand as a value, rather than + // a pointer to the stack value + if (payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { + return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)}); + }; + try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 }); + } + } else if (payload_ty.isSlice(zcu)) { + switch (cg.ptr_size) { + .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }), + .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }), + } + } + + // Compare the null value with '0' + try cg.addImm32(0); + try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); + + return .stack; +} + +fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const opt_ty = cg.typeOf(ty_op.operand); + const payload_ty = cg.typeOfIndex(inst); + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + return cg.finishAir(inst, .none, &.{ty_op.operand}); + } + + const result = result: { + const operand = try cg.resolveInst(ty_op.operand); + if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand); + + if (isByRef(payload_ty, zcu, cg.target)) { + break :result try cg.buildPointerOffset(operand, 0, .new); + } + + break :result try cg.load(operand, payload_ty, 0); + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); + + const result = result: { + const payload_ty = opt_ty.optionalChild(zcu); + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu) or opt_ty.optionalReprIsPayload(zcu)) { + break :result cg.reuseOperand(ty_op.operand, operand); + } + + break :result try cg.buildPointerOffset(operand, 0, .new); + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); + const payload_ty = opt_ty.optionalChild(zcu); + + if (opt_ty.optionalReprIsPayload(zcu)) { + return cg.finishAir(inst, operand, &.{ty_op.operand}); + } + + const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { + return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)}); + }; + + try cg.emitWValue(operand); + try cg.addImm32(1); + try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 }); + + const result = try cg.buildPointerOffset(operand, 0, .new); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const payload_ty = cg.typeOf(ty_op.operand); + const pt = cg.pt; + const zcu = pt.zcu; + + const result = result: { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + const non_null_bit = try cg.allocStack(Type.u1); + try cg.emitWValue(non_null_bit); + try cg.addImm32(1); + try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 }); + break :result non_null_bit; + } + + const operand = try cg.resolveInst(ty_op.operand); + const op_ty = cg.typeOfIndex(inst); + if (op_ty.optionalReprIsPayload(zcu)) { + break :result cg.reuseOperand(ty_op.operand, operand); + } + const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { + return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)}); + }; + + // Create optional type, set the non-null bit, and store the operand inside the optional type + const result_ptr = try cg.allocStack(op_ty); + try cg.emitWValue(result_ptr); + try cg.addImm32(1); + try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 }); + + const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new); + try cg.store(payload_ptr, operand, payload_ty, 0); + break :result result_ptr; + }; + + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const slice_ty = cg.typeOfIndex(inst); + + const slice = try cg.allocStack(slice_ty); + try cg.store(slice, lhs, Type.usize, 0); + try cg.store(slice, rhs, Type.usize, cg.ptrSize()); + + return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand}); +} + +fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const slice_ty = cg.typeOf(bin_op.lhs); + const slice = try cg.resolveInst(bin_op.lhs); + const index = try cg.resolveInst(bin_op.rhs); + const elem_ty = slice_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + // load pointer onto stack + _ = try cg.load(slice, Type.usize, 0); + + // calculate index into slice + try cg.emitWValue(index); + try cg.addImm32(@intCast(elem_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + + const elem_result = if (isByRef(elem_ty, zcu, cg.target)) + .stack + else + try cg.load(.stack, elem_ty, 0); + + return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const elem_ty = ty_pl.ty.toType().childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + const slice = try cg.resolveInst(bin_op.lhs); + const index = try cg.resolveInst(bin_op.rhs); + + _ = try cg.load(slice, Type.usize, 0); + + // calculate index into slice + try cg.emitWValue(index); + try cg.addImm32(@intCast(elem_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand}); +} + +fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue { + const ptr = try cg.load(operand, Type.usize, 0); + return ptr.toLocal(cg, Type.usize); +} + +fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue { + const len = try cg.load(operand, Type.usize, cg.ptrSize()); + return len.toLocal(cg, Type.usize); +} + +fn airTrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const wanted_ty: Type = ty_op.ty.toType(); + const op_ty = cg.typeOf(ty_op.operand); + const zcu = cg.pt.zcu; + + if (wanted_ty.zigTypeTag(zcu) == .vector or op_ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: trunc for vectors", .{}); + } + + const result = if (op_ty.bitSize(zcu) == wanted_ty.bitSize(zcu)) + cg.reuseOperand(ty_op.operand, operand) + else + try cg.trunc(operand, wanted_ty, op_ty); + + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// Truncates a given operand to a given type, discarding any overflown bits. +/// NOTE: Resulting value is left on the stack. +fn trunc(cg: *CodeGen, operand: WValue, wanted_ty: Type, given_ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + const given_bits = @as(u16, @intCast(given_ty.bitSize(zcu))); + if (toWasmBits(given_bits) == null) { + return cg.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{given_bits}); + } + + var result = try cg.intcast(operand, given_ty, wanted_ty); + const wanted_bits = @as(u16, @intCast(wanted_ty.bitSize(zcu))); + const wasm_bits = toWasmBits(wanted_bits).?; + if (wasm_bits != wanted_bits) { + result = try cg.wrapOperand(result, wanted_ty); + } + return result; +} + +fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const array_ty = cg.typeOf(ty_op.operand).childType(zcu); + const slice_ty = ty_op.ty.toType(); + + // create a slice on the stack + const slice_local = try cg.allocStack(slice_ty); + + // store the array ptr in the slice + if (array_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + try cg.store(slice_local, operand, Type.usize, 0); + } + + // store the length of the array in the slice + const array_len: u32 = @intCast(array_ty.arrayLen(zcu)); + try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize()); + + return cg.finishAir(inst, slice_local, &.{ty_op.operand}); +} + +fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ptr_ty = cg.typeOf(bin_op.lhs); + const ptr = try cg.resolveInst(bin_op.lhs); + const index = try cg.resolveInst(bin_op.rhs); + const elem_ty = ptr_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + // load pointer onto the stack + if (ptr_ty.isSlice(zcu)) { + _ = try cg.load(ptr, Type.usize, 0); + } else { + try cg.lowerToStack(ptr); + } + + // calculate index into slice + try cg.emitWValue(index); + try cg.addImm32(@intCast(elem_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + + const elem_result = if (isByRef(elem_ty, zcu, cg.target)) + .stack + else + try cg.load(.stack, elem_ty, 0); + + return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const ptr_ty = cg.typeOf(bin_op.lhs); + const elem_ty = ty_pl.ty.toType().childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + const ptr = try cg.resolveInst(bin_op.lhs); + const index = try cg.resolveInst(bin_op.rhs); + + // load pointer onto the stack + if (ptr_ty.isSlice(zcu)) { + _ = try cg.load(ptr, Type.usize, 0); + } else { + try cg.lowerToStack(ptr); + } + + // calculate index into ptr + try cg.emitWValue(index); + try cg.addImm32(@intCast(elem_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { + const zcu = cg.pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const ptr = try cg.resolveInst(bin_op.lhs); + const offset = try cg.resolveInst(bin_op.rhs); + const ptr_ty = cg.typeOf(bin_op.lhs); + const pointee_ty = switch (ptr_ty.ptrSize(zcu)) { + .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type + else => ptr_ty.childType(zcu), + }; + + const valtype = typeToValtype(Type.usize, zcu, cg.target); + const mul_opcode = buildOpcode(.{ .valtype1 = valtype, .op = .mul }); + const bin_opcode = buildOpcode(.{ .valtype1 = valtype, .op = op }); + + try cg.lowerToStack(ptr); + try cg.emitWValue(offset); + try cg.addImm32(@intCast(pointee_ty.abiSize(zcu))); + try cg.addTag(Mir.Inst.Tag.fromOpcode(mul_opcode)); + try cg.addTag(Mir.Inst.Tag.fromOpcode(bin_opcode)); + + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ptr = try cg.resolveInst(bin_op.lhs); + const ptr_ty = cg.typeOf(bin_op.lhs); + const value = try cg.resolveInst(bin_op.rhs); + const len = switch (ptr_ty.ptrSize(zcu)) { + .slice => try cg.sliceLen(ptr), + .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }), + .c, .many => unreachable, + }; + + const elem_ty = if (ptr_ty.ptrSize(zcu) == .one) + ptr_ty.childType(zcu).childType(zcu) + else + ptr_ty.childType(zcu); + + if (!safety and bin_op.rhs == .undef) { + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); + } + + const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty); + try cg.memset(elem_ty, dst_ptr, len, value); + + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); +} + +/// Sets a region of memory at `ptr` to the value of `value` +/// When the user has enabled the bulk_memory feature, we lower +/// this to wasm's memset instruction. When the feature is not present, +/// we implement it manually. +fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void { + const zcu = cg.pt.zcu; + const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); + + // When bulk_memory is enabled, we lower it to wasm's memset instruction. + // If not, we lower it ourselves. + if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) { + const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0); + + if (!len0_ok) { + try cg.startBlock(.block, .empty); + + // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is + // out of memory bounds. This can easily happen in Zig in a case such as: + // + // const ptr: [*]u8 = undefined; + // var len: usize = runtime_zero(); + // @memset(ptr[0..len], 42); + // + // So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design. + try cg.emitWValue(len); + try cg.addTag(.i32_eqz); + try cg.addLabel(.br_if, 0); + } + + try cg.lowerToStack(ptr); + try cg.emitWValue(value); + try cg.emitWValue(len); + try cg.addExtended(.memory_fill); + + if (!len0_ok) { + try cg.endBlock(); + } + + return; + } + + const final_len: WValue = switch (len) { + .imm32 => |val| .{ .imm32 = val * abi_size }, + .imm64 => |val| .{ .imm64 = val * abi_size }, + else => if (abi_size != 1) blk: { + const new_len = try cg.ensureAllocLocal(Type.usize); + try cg.emitWValue(len); + switch (cg.ptr_size) { + .wasm32 => { + try cg.emitWValue(.{ .imm32 = abi_size }); + try cg.addTag(.i32_mul); + }, + .wasm64 => { + try cg.emitWValue(.{ .imm64 = abi_size }); + try cg.addTag(.i64_mul); + }, + } + try cg.addLocal(.local_set, new_len.local.value); + break :blk new_len; + } else len, + }; + + var end_ptr = try cg.allocLocal(Type.usize); + defer end_ptr.free(cg); + var new_ptr = try cg.buildPointerOffset(ptr, 0, .new); + defer new_ptr.free(cg); + + // get the loop conditional: if current pointer address equals final pointer's address + try cg.lowerToStack(ptr); + try cg.emitWValue(final_len); + switch (cg.ptr_size) { + .wasm32 => try cg.addTag(.i32_add), + .wasm64 => try cg.addTag(.i64_add), + } + try cg.addLocal(.local_set, end_ptr.local.value); + + // outer block to jump to when loop is done + try cg.startBlock(.block, .empty); + try cg.startBlock(.loop, .empty); + + // check for condition for loop end + try cg.emitWValue(new_ptr); + try cg.emitWValue(end_ptr); + switch (cg.ptr_size) { + .wasm32 => try cg.addTag(.i32_eq), + .wasm64 => try cg.addTag(.i64_eq), + } + try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished) + + // store the value at the current position of the pointer + try cg.store(new_ptr, value, elem_ty, 0); + + // move the pointer to the next element + try cg.emitWValue(new_ptr); + switch (cg.ptr_size) { + .wasm32 => { + try cg.emitWValue(.{ .imm32 = abi_size }); + try cg.addTag(.i32_add); + }, + .wasm64 => { + try cg.emitWValue(.{ .imm64 = abi_size }); + try cg.addTag(.i64_add); + }, + } + try cg.addLocal(.local_set, new_ptr.local.value); + + // end of loop + try cg.addLabel(.br, 0); // jump to start of loop + try cg.endBlock(); + try cg.endBlock(); +} + +fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const array_ty = cg.typeOf(bin_op.lhs); + const array = try cg.resolveInst(bin_op.lhs); + const index = try cg.resolveInst(bin_op.rhs); + const elem_ty = array_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + if (isByRef(array_ty, zcu, cg.target)) { + try cg.lowerToStack(array); + try cg.emitWValue(index); + try cg.addImm32(@intCast(elem_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + } else { + assert(array_ty.zigTypeTag(zcu) == .vector); + + switch (index) { + inline .imm32, .imm64 => |lane| { + const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) { + 8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u, + 16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u, + 32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane, + 64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane, + else => unreachable, + }; + + var operands = [_]u32{ @intFromEnum(opcode), @as(u8, @intCast(lane)) }; + + try cg.emitWValue(array); + + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + try cg.mir_extra.appendSlice(cg.gpa, &operands); + try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); + + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); + }, + else => { + const stack_vec = try cg.allocStack(array_ty); + try cg.store(stack_vec, array, array_ty, 0); + + // Is a non-unrolled vector (v128) + try cg.lowerToStack(stack_vec); + try cg.emitWValue(index); + try cg.addImm32(@intCast(elem_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + }, + } + } + + const elem_result = if (isByRef(elem_ty, zcu, cg.target)) + .stack + else + try cg.load(.stack, elem_ty, 0); + + return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airIntFromFloat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const op_ty = cg.typeOf(ty_op.operand); + const op_bits = op_ty.floatBits(cg.target); + + const dest_ty = cg.typeOfIndex(inst); + const dest_info = dest_ty.intInfo(zcu); + + if (dest_info.bits > 128) { + return cg.fail("TODO: intFromFloat for integers/floats with bitsize {}", .{dest_info.bits}); + } + + if ((op_bits != 32 and op_bits != 64) or dest_info.bits > 64) { + const dest_bitsize = if (dest_info.bits <= 32) 32 else std.math.ceilPowerOfTwoAssert(u16, dest_info.bits); + + const intrinsic = switch (dest_info.signedness) { + inline .signed, .unsigned => |ct_s| switch (op_bits) { + inline 16, 32, 64, 80, 128 => |ct_op_bits| switch (dest_bitsize) { + inline 32, 64, 128 => |ct_dest_bits| @field( + Mir.Intrinsic, + "__fix" ++ switch (ct_s) { + .signed => "", + .unsigned => "uns", + } ++ + compilerRtFloatAbbrev(ct_op_bits) ++ "f" ++ + compilerRtIntAbbrev(ct_dest_bits) ++ "i", + ), + else => unreachable, + }, + else => unreachable, + }, + }; + const result = try cg.callIntrinsic(intrinsic, &.{op_ty.ip_index}, dest_ty, &.{operand}); + return cg.finishAir(inst, result, &.{ty_op.operand}); + } + + try cg.emitWValue(operand); + const op = buildOpcode(.{ + .op = .trunc, + .valtype1 = typeToValtype(dest_ty, zcu, cg.target), + .valtype2 = typeToValtype(op_ty, zcu, cg.target), + .signedness = dest_info.signedness, + }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(op)); + const result = try cg.wrapOperand(.stack, dest_ty); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airFloatFromInt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const op_ty = cg.typeOf(ty_op.operand); + const op_info = op_ty.intInfo(zcu); + + const dest_ty = cg.typeOfIndex(inst); + const dest_bits = dest_ty.floatBits(cg.target); + + if (op_info.bits > 128) { + return cg.fail("TODO: floatFromInt for integers/floats with bitsize {d} bits", .{op_info.bits}); + } + + if (op_info.bits > 64 or (dest_bits > 64 or dest_bits < 32)) { + const op_bitsize = if (op_info.bits <= 32) 32 else std.math.ceilPowerOfTwoAssert(u16, op_info.bits); + + const intrinsic = switch (op_info.signedness) { + inline .signed, .unsigned => |ct_s| switch (op_bitsize) { + inline 32, 64, 128 => |ct_int_bits| switch (dest_bits) { + inline 16, 32, 64, 80, 128 => |ct_float_bits| @field( + Mir.Intrinsic, + "__float" ++ switch (ct_s) { + .signed => "", + .unsigned => "un", + } ++ + compilerRtIntAbbrev(ct_int_bits) ++ "i" ++ + compilerRtFloatAbbrev(ct_float_bits) ++ "f", + ), + else => unreachable, + }, + else => unreachable, + }, + }; + + const result = try cg.callIntrinsic(intrinsic, &.{op_ty.ip_index}, dest_ty, &.{operand}); + return cg.finishAir(inst, result, &.{ty_op.operand}); + } + + try cg.emitWValue(operand); + const op = buildOpcode(.{ + .op = .convert, + .valtype1 = typeToValtype(dest_ty, zcu, cg.target), + .valtype2 = typeToValtype(op_ty, zcu, cg.target), + .signedness = op_info.signedness, + }); + try cg.addTag(Mir.Inst.Tag.fromOpcode(op)); + + return cg.finishAir(inst, .stack, &.{ty_op.operand}); +} + +fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try cg.resolveInst(ty_op.operand); + const ty = cg.typeOfIndex(inst); + const elem_ty = ty.childType(zcu); + + if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: { + switch (operand) { + // when the operand lives in the linear memory section, we can directly + // load and splat the value at once. Meaning we do not first have to load + // the scalar value onto the stack. + .stack_offset, .nav_ref, .uav_ref => { + const opcode = switch (elem_ty.bitSize(zcu)) { + 8 => @intFromEnum(std.wasm.SimdOpcode.v128_load8_splat), + 16 => @intFromEnum(std.wasm.SimdOpcode.v128_load16_splat), + 32 => @intFromEnum(std.wasm.SimdOpcode.v128_load32_splat), + 64 => @intFromEnum(std.wasm.SimdOpcode.v128_load64_splat), + else => break :blk, // Cannot make use of simd-instructions + }; + try cg.emitWValue(operand); + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + // stores as := opcode, offset, alignment (opcode::memarg) + try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ + opcode, + operand.offset(), + @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?), + }); + try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); + return cg.finishAir(inst, .stack, &.{ty_op.operand}); + }, + .local => { + const opcode = switch (elem_ty.bitSize(zcu)) { + 8 => @intFromEnum(std.wasm.SimdOpcode.i8x16_splat), + 16 => @intFromEnum(std.wasm.SimdOpcode.i16x8_splat), + 32 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i32x4_splat) else @intFromEnum(std.wasm.SimdOpcode.f32x4_splat), + 64 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i64x2_splat) else @intFromEnum(std.wasm.SimdOpcode.f64x2_splat), + else => break :blk, // Cannot make use of simd-instructions + }; + try cg.emitWValue(operand); + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + try cg.mir_extra.append(cg.gpa, opcode); + try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); + return cg.finishAir(inst, .stack, &.{ty_op.operand}); + }, + else => unreachable, + } + } + const elem_size = elem_ty.bitSize(zcu); + const vector_len = @as(usize, @intCast(ty.vectorLen(zcu))); + if ((!std.math.isPowerOfTwo(elem_size) or elem_size % 8 != 0) and vector_len > 1) { + return cg.fail("TODO: WebAssembly `@splat` for arbitrary element bitsize {d}", .{elem_size}); + } + + const result = try cg.allocStack(ty); + const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); + var index: usize = 0; + var offset: u32 = 0; + while (index < vector_len) : (index += 1) { + try cg.store(result, operand, elem_ty, offset); + offset += elem_byte_size; + } + + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const operand = try cg.resolveInst(pl_op.operand); + + _ = operand; + return cg.fail("TODO: Implement wasm airSelect", .{}); +} + +fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + + const unwrapped = cg.air.unwrapShuffleOne(zcu, inst); + const result_ty = unwrapped.result_ty; + const mask = unwrapped.mask; + const operand = try cg.resolveInst(unwrapped.operand); + + const elem_ty = result_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + // TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were + // to lower the comptime-known operands to a non-by-ref vector value. + + // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible. + // I tried to fix it, but I couldn't make much sense of how this backend handles memory. + if (!isByRef(result_ty, zcu, cg.target) or + !isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{}); + + const dest_alloc = try cg.allocStack(result_ty); + for (mask, 0..) |mask_elem, out_idx| { + try cg.emitWValue(dest_alloc); + const elem_val = switch (mask_elem.unwrap()) { + .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)), + .value => |val| try cg.lowerConstant(.fromInterned(val), elem_ty), + }; + try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx)); + } + return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand}); +} + +fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + + const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst); + const result_ty = unwrapped.result_ty; + const mask = unwrapped.mask; + const operand_a = try cg.resolveInst(unwrapped.operand_a); + const operand_b = try cg.resolveInst(unwrapped.operand_b); + + const a_ty = cg.typeOf(unwrapped.operand_a); + const b_ty = cg.typeOf(unwrapped.operand_b); + const elem_ty = result_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + + // WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8 + // and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise, + // we fall back to a naive loop lowering. + if (!isByRef(a_ty, zcu, cg.target) and + !isByRef(b_ty, zcu, cg.target) and + !isByRef(result_ty, zcu, cg.target) and + elem_ty.bitSize(zcu) % 8 == 0) + { + var lane_map: [16]u8 align(4) = undefined; + const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8); + for (mask, 0..) |mask_elem, out_idx| { + const out_first_lane = out_idx * lanes_per_elem; + const in_first_lane = switch (mask_elem.unwrap()) { + .a_elem => |i| i * lanes_per_elem, + .b_elem => |i| i * lanes_per_elem + 16, + .undef => 0, // doesn't matter + }; + for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| { + out.* = @intCast(in); + } + } + try cg.emitWValue(operand_a); + try cg.emitWValue(operand_b); + const extra_index: u32 = @intCast(cg.mir_extra.items.len); + try cg.mir_extra.appendSlice(cg.gpa, &.{ + @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle), + @bitCast(lane_map[0..4].*), + @bitCast(lane_map[4..8].*), + @bitCast(lane_map[8..12].*), + @bitCast(lane_map[12..].*), + }); + try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); + return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b }); + } + + // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible. + // I tried to fix it, but I couldn't make much sense of how this backend handles memory. + if (!isByRef(result_ty, zcu, cg.target) or + !isByRef(a_ty, zcu, cg.target) or + !isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{}); + + const dest_alloc = try cg.allocStack(result_ty); + for (mask, 0..) |mask_elem, out_idx| { + try cg.emitWValue(dest_alloc); + const elem_val = switch (mask_elem.unwrap()) { + .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)), + .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)), + .undef => try cg.emitUndefined(elem_ty), + }; + try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx)); + } + return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b }); +} + +fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const reduce = cg.air.instructions.items(.data)[@intFromEnum(inst)].reduce; + const operand = try cg.resolveInst(reduce.operand); + + _ = operand; + return cg.fail("TODO: Implement wasm airReduce", .{}); +} + +fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const result_ty = cg.typeOfIndex(inst); + const len = @as(usize, @intCast(result_ty.arrayLen(zcu))); + const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]); + + const result: WValue = result_value: { + switch (result_ty.zigTypeTag(zcu)) { + .array => { + const result = try cg.allocStack(result_ty); + const elem_ty = result_ty.childType(zcu); + const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); + const sentinel = result_ty.sentinel(zcu); + + // When the element type is by reference, we must copy the entire + // value. It is therefore safer to move the offset pointer and store + // each value individually, instead of using store offsets. + if (isByRef(elem_ty, zcu, cg.target)) { + // copy stack pointer into a temporary local, which is + // moved for each element to store each value in the right position. + const offset = try cg.buildPointerOffset(result, 0, .new); + for (elements, 0..) |elem, elem_index| { + const elem_val = try cg.resolveInst(elem); + try cg.store(offset, elem_val, elem_ty, 0); + + if (elem_index < elements.len - 1 or sentinel != null) { + _ = try cg.buildPointerOffset(offset, elem_size, .modify); + } + } + if (sentinel) |s| { + const val = try cg.resolveValue(s); + try cg.store(offset, val, elem_ty, 0); + } + } else { + var offset: u32 = 0; + for (elements) |elem| { + const elem_val = try cg.resolveInst(elem); + try cg.store(result, elem_val, elem_ty, offset); + offset += elem_size; + } + if (sentinel) |s| { + const val = try cg.resolveValue(s); + try cg.store(result, val, elem_ty, offset); + } + } + break :result_value result; + }, + .@"struct" => switch (result_ty.containerLayout(zcu)) { + .@"packed" => { + if (isByRef(result_ty, zcu, cg.target)) { + return cg.fail("TODO: airAggregateInit for packed structs larger than 64 bits", .{}); + } + const packed_struct = zcu.typeToPackedStruct(result_ty).?; + const field_types = packed_struct.field_types; + const backing_type = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); + + // ensure the result is zero'd + const result = try cg.allocLocal(backing_type); + if (backing_type.bitSize(zcu) <= 32) + try cg.addImm32(0) + else + try cg.addImm64(0); + try cg.addLocal(.local_set, result.local.value); + + var current_bit: u16 = 0; + for (elements, 0..) |elem, elem_index| { + const field_ty = Type.fromInterned(field_types.get(ip)[elem_index]); + if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; + + const shift_val: WValue = if (backing_type.bitSize(zcu) <= 32) + .{ .imm32 = current_bit } + else + .{ .imm64 = current_bit }; + + const value = try cg.resolveInst(elem); + const value_bit_size: u16 = @intCast(field_ty.bitSize(zcu)); + const int_ty = try pt.intType(.unsigned, value_bit_size); + + // load our current result on stack so we can perform all transformations + // using only stack values. Saving the cost of loads and stores. + try cg.emitWValue(result); + const bitcasted = try cg.bitcast(int_ty, field_ty, value); + const extended_val = try cg.intcast(bitcasted, int_ty, backing_type); + // no need to shift any values when the current offset is 0 + const shifted = if (current_bit != 0) shifted: { + break :shifted try cg.binOp(extended_val, shift_val, backing_type, .shl); + } else extended_val; + // we ignore the result as we keep it on the stack to assign it directly to `result` + _ = try cg.binOp(.stack, shifted, backing_type, .@"or"); + try cg.addLocal(.local_set, result.local.value); + current_bit += value_bit_size; + } + break :result_value result; + }, + else => { + const result = try cg.allocStack(result_ty); + const offset = try cg.buildPointerOffset(result, 0, .new); // pointer to offset + var prev_field_offset: u64 = 0; + for (elements, 0..) |elem, elem_index| { + if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue; + + const elem_ty = result_ty.fieldType(elem_index, zcu); + const field_offset = result_ty.structFieldOffset(elem_index, zcu); + _ = try cg.buildPointerOffset(offset, @intCast(field_offset - prev_field_offset), .modify); + prev_field_offset = field_offset; + + const value = try cg.resolveInst(elem); + try cg.store(offset, value, elem_ty, 0); + } + + break :result_value result; + }, + }, + .vector => return cg.fail("TODO: Wasm backend: implement airAggregateInit for vectors", .{}), + else => unreachable, + } + }; + + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); + @memcpy(buf[0..elements.len], elements); + return cg.finishAir(inst, result, &buf); + } + var bt = try cg.iterateBigTomb(inst, elements.len); + for (elements) |arg| bt.feed(arg); + return bt.finishAir(result); +} + +fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; + + const result = result: { + const union_ty = cg.typeOfIndex(inst); + const layout = union_ty.unionGetLayout(zcu); + const union_obj = zcu.typeToUnion(union_ty).?; + const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]); + const field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index]; + + const tag_int = blk: { + const tag_ty = union_ty.unionTagTypeHypothetical(zcu); + const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?; + const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index); + break :blk try cg.lowerConstant(tag_val, tag_ty); + }; + if (layout.payload_size == 0) { + if (layout.tag_size == 0) { + break :result .none; + } + assert(!isByRef(union_ty, zcu, cg.target)); + break :result tag_int; + } + + if (isByRef(union_ty, zcu, cg.target)) { + const result_ptr = try cg.allocStack(union_ty); + const payload = try cg.resolveInst(extra.init); + if (layout.tag_align.compare(.gte, layout.payload_align)) { + if (isByRef(field_ty, zcu, cg.target)) { + const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new); + try cg.store(payload_ptr, payload, field_ty, 0); + } else { + try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size)); + } + + if (layout.tag_size > 0) { + try cg.store(result_ptr, tag_int, Type.fromInterned(union_obj.enum_tag_ty), 0); + } + } else { + try cg.store(result_ptr, payload, field_ty, 0); + if (layout.tag_size > 0) { + try cg.store( + result_ptr, + tag_int, + Type.fromInterned(union_obj.enum_tag_ty), + @intCast(layout.payload_size), + ); + } + } + break :result result_ptr; + } else { + const operand = try cg.resolveInst(extra.init); + const union_int_type = try pt.intType(.unsigned, @as(u16, @intCast(union_ty.bitSize(zcu)))); + if (field_ty.zigTypeTag(zcu) == .float) { + const int_type = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu))); + const bitcasted = try cg.bitcast(field_ty, int_type, operand); + break :result try cg.trunc(bitcasted, int_type, union_int_type); + } else if (field_ty.isPtrAtRuntime(zcu)) { + const int_type = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu))); + break :result try cg.intcast(operand, int_type, union_int_type); + } + break :result try cg.intcast(operand, field_ty, union_int_type); + } + }; + + return cg.finishAir(inst, result, &.{extra.init}); +} + +fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const prefetch = cg.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; + return cg.finishAir(inst, .none, &.{prefetch.ptr}); +} + +fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + + try cg.addLabel(.memory_size, pl_op.payload); + return cg.finishAir(inst, .stack, &.{pl_op.operand}); +} + +fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void { + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + + const operand = try cg.resolveInst(pl_op.operand); + try cg.emitWValue(operand); + try cg.addLabel(.memory_grow, pl_op.payload); + return cg.finishAir(inst, .stack, &.{pl_op.operand}); +} + +fn cmpOptionals(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue { + const zcu = cg.pt.zcu; + assert(operand_ty.hasRuntimeBitsIgnoreComptime(zcu)); + assert(op == .eq or op == .neq); + const payload_ty = operand_ty.optionalChild(zcu); + assert(!isByRef(payload_ty, zcu, cg.target)); + + var result = try cg.allocLocal(Type.i32); + defer result.free(cg); + + var lhs_null = try cg.allocLocal(Type.i32); + defer lhs_null.free(cg); + + try cg.startBlock(.block, .empty); + + try cg.addImm32(if (op == .eq) 0 else 1); + try cg.addLocal(.local_set, result.local.value); + + _ = try cg.isNull(lhs, operand_ty, .i32_eq); + try cg.addLocal(.local_tee, lhs_null.local.value); + _ = try cg.isNull(rhs, operand_ty, .i32_eq); + try cg.addTag(.i32_ne); + try cg.addLabel(.br_if, 0); // only one is null + + try cg.addImm32(if (op == .eq) 1 else 0); + try cg.addLocal(.local_set, result.local.value); + + try cg.addLocal(.local_get, lhs_null.local.value); + try cg.addLabel(.br_if, 0); // both are null + + _ = try cg.load(lhs, payload_ty, 0); + _ = try cg.load(rhs, payload_ty, 0); + _ = try cg.cmp(.stack, .stack, payload_ty, op); + try cg.addLocal(.local_set, result.local.value); + + try cg.endBlock(); + + try cg.addLocal(.local_get, result.local.value); + + return .stack; +} + +/// Compares big integers by checking both its high bits and low bits. +/// NOTE: Leaves the result of the comparison on top of the stack. +/// TODO: Lower this to compiler_rt call when bitsize > 128 +fn cmpBigInt(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue { + const zcu = cg.pt.zcu; + assert(operand_ty.abiSize(zcu) >= 16); + assert(!(lhs != .stack and rhs == .stack)); + if (operand_ty.bitSize(zcu) > 128) { + return cg.fail("TODO: Support cmpBigInt for integer bitsize: '{d}'", .{operand_ty.bitSize(zcu)}); + } + + var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64); + defer lhs_msb.free(cg); + var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64); + defer rhs_msb.free(cg); + + switch (op) { + .eq, .neq => { + const xor_high = try cg.binOp(lhs_msb, rhs_msb, Type.u64, .xor); + const lhs_lsb = try cg.load(lhs, Type.u64, 0); + const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const xor_low = try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, .xor); + const or_result = try cg.binOp(xor_high, xor_low, Type.u64, .@"or"); + + switch (op) { + .eq => return cg.cmp(or_result, .{ .imm64 = 0 }, Type.u64, .eq), + .neq => return cg.cmp(or_result, .{ .imm64 = 0 }, Type.u64, .neq), + else => unreachable, + } + }, + else => { + const ty = if (operand_ty.isSignedInt(zcu)) Type.i64 else Type.u64; + // leave those value on top of the stack for '.select' + const lhs_lsb = try cg.load(lhs, Type.u64, 0); + const rhs_lsb = try cg.load(rhs, Type.u64, 0); + _ = try cg.cmp(lhs_lsb, rhs_lsb, Type.u64, op); + _ = try cg.cmp(lhs_msb, rhs_msb, ty, op); + _ = try cg.cmp(lhs_msb, rhs_msb, ty, .eq); + try cg.addTag(.select); + }, + } + + return .stack; +} + +fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const un_ty = cg.typeOf(bin_op.lhs).childType(zcu); + const tag_ty = cg.typeOf(bin_op.rhs); + const layout = un_ty.unionGetLayout(zcu); + if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); + + const union_ptr = try cg.resolveInst(bin_op.lhs); + const new_tag = try cg.resolveInst(bin_op.rhs); + if (layout.payload_size == 0) { + try cg.store(union_ptr, new_tag, tag_ty, 0); + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); + } + + // when the tag alignment is smaller than the payload, the field will be stored + // after the payload. + const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: { + break :blk @intCast(layout.payload_size); + } else 0; + try cg.store(union_ptr, new_tag, tag_ty, offset); + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const un_ty = cg.typeOf(ty_op.operand); + const tag_ty = cg.typeOfIndex(inst); + const layout = un_ty.unionGetLayout(zcu); + if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand}); + + const operand = try cg.resolveInst(ty_op.operand); + // when the tag alignment is smaller than the payload, the field will be stored + // after the payload. + const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) + @intCast(layout.payload_size) + else + 0; + const result = try cg.load(operand, tag_ty, offset); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airFpext(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dest_ty = cg.typeOfIndex(inst); + const operand = try cg.resolveInst(ty_op.operand); + const result = try cg.fpext(operand, cg.typeOf(ty_op.operand), dest_ty); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// Extends a float from a given `Type` to a larger wanted `Type`, leaving the +/// result on the stack. +fn fpext(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue { + const given_bits = given.floatBits(cg.target); + const wanted_bits = wanted.floatBits(cg.target); + + const intrinsic: Mir.Intrinsic = switch (given_bits) { + 16 => switch (wanted_bits) { + 32 => { + assert(.stack == try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand})); + return .stack; + }, + 64 => { + assert(.stack == try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand})); + try cg.addTag(.f64_promote_f32); + return .stack; + }, + 80 => .__extendhfxf2, + 128 => .__extendhftf2, + else => unreachable, + }, + 32 => switch (wanted_bits) { + 64 => { + try cg.emitWValue(operand); + try cg.addTag(.f64_promote_f32); + return .stack; + }, + 80 => .__extendsfxf2, + 128 => .__extendsftf2, + else => unreachable, + }, + 64 => switch (wanted_bits) { + 80 => .__extenddfxf2, + 128 => .__extenddftf2, + else => unreachable, + }, + 80 => switch (wanted_bits) { + 128 => .__extendxftf2, + else => unreachable, + }, + else => unreachable, + }; + return cg.callIntrinsic(intrinsic, &.{given.ip_index}, wanted, &.{operand}); +} + +fn airFptrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dest_ty = cg.typeOfIndex(inst); + const operand = try cg.resolveInst(ty_op.operand); + const result = try cg.fptrunc(operand, cg.typeOf(ty_op.operand), dest_ty); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// Truncates a float from a given `Type` to its wanted `Type`, leaving the +/// result on the stack. +fn fptrunc(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue { + const given_bits = given.floatBits(cg.target); + const wanted_bits = wanted.floatBits(cg.target); + + const intrinsic: Mir.Intrinsic = switch (given_bits) { + 32 => switch (wanted_bits) { + 16 => { + return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand}); + }, + else => unreachable, + }, + 64 => switch (wanted_bits) { + 16 => { + try cg.emitWValue(operand); + try cg.addTag(.f32_demote_f64); + return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack}); + }, + 32 => { + try cg.emitWValue(operand); + try cg.addTag(.f32_demote_f64); + return .stack; + }, + else => unreachable, + }, + 80 => switch (wanted_bits) { + 16 => .__truncxfhf2, + 32 => .__truncxfsf2, + 64 => .__truncxfdf2, + else => unreachable, + }, + 128 => switch (wanted_bits) { + 16 => .__trunctfhf2, + 32 => .__trunctfsf2, + 64 => .__trunctfdf2, + 80 => .__trunctfxf2, + else => unreachable, + }, + else => unreachable, + }; + return cg.callIntrinsic(intrinsic, &.{given.ip_index}, wanted, &.{operand}); +} + +fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu); + const payload_ty = err_set_ty.errorUnionPayload(zcu); + const operand = try cg.resolveInst(ty_op.operand); + + // set error-tag to '0' to annotate error union is non-error + try cg.store( + operand, + .{ .imm32 = 0 }, + Type.anyerror, + @intCast(errUnionErrorOffset(payload_ty, zcu)), + ); + + const result = result: { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :result cg.reuseOperand(ty_op.operand, operand); + } + + break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new); + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + + const field_ptr = try cg.resolveInst(extra.field_ptr); + const parent_ptr_ty = cg.typeOfIndex(inst); + const parent_ty = parent_ptr_ty.childType(zcu); + const field_ptr_ty = cg.typeOf(extra.field_ptr); + const field_index = extra.field_index; + const field_offset = switch (parent_ty.containerLayout(zcu)) { + .auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu), + .@"packed" => offset: { + const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset; + const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0; + const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset; + break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8); + }, + }; + + const result = if (field_offset != 0) result: { + const base = try cg.buildPointerOffset(field_ptr, 0, .new); + try cg.addLocal(.local_get, base.local.value); + try cg.addImm32(@intCast(field_offset)); + try cg.addTag(.i32_sub); + try cg.addLocal(.local_set, base.local.value); + break :result base; + } else cg.reuseOperand(extra.field_ptr, field_ptr); + + return cg.finishAir(inst, result, &.{extra.field_ptr}); +} + +fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + if (ptr_ty.isSlice(zcu)) { + return cg.slicePtr(ptr); + } else { + return ptr; + } +} + +fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dst = try cg.resolveInst(bin_op.lhs); + const dst_ty = cg.typeOf(bin_op.lhs); + const ptr_elem_ty = dst_ty.childType(zcu); + const src = try cg.resolveInst(bin_op.rhs); + const src_ty = cg.typeOf(bin_op.rhs); + const len = switch (dst_ty.ptrSize(zcu)) { + .slice => blk: { + const slice_len = try cg.sliceLen(dst); + if (ptr_elem_ty.abiSize(zcu) != 1) { + try cg.emitWValue(slice_len); + try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) }); + try cg.addTag(.i32_mul); + try cg.addLocal(.local_set, slice_len.local.value); + } + break :blk slice_len; + }, + .one => @as(WValue, .{ + .imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))), + }), + .c, .many => unreachable, + }; + const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty); + const src_ptr = try cg.sliceOrArrayPtr(src, src_ty); + try cg.memcpy(dst_ptr, src_ptr, len); + + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + // TODO: Implement this properly once stack serialization is solved + return cg.finishAir(inst, switch (cg.ptr_size) { + .wasm32 => .{ .imm32 = 0 }, + .wasm64 => .{ .imm64 = 0 }, + }, &.{}); +} + +fn airPopcount(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const op_ty = cg.typeOf(ty_op.operand); + + if (op_ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: Implement @popCount for vectors", .{}); + } + + const int_info = op_ty.intInfo(zcu); + const bits = int_info.bits; + const wasm_bits = toWasmBits(bits) orelse { + return cg.fail("TODO: Implement @popCount for integers with bitsize '{d}'", .{bits}); + }; + + switch (wasm_bits) { + 32 => { + try cg.emitWValue(operand); + if (op_ty.isSignedInt(zcu) and bits != wasm_bits) { + _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits)); + } + try cg.addTag(.i32_popcnt); + }, + 64 => { + try cg.emitWValue(operand); + if (op_ty.isSignedInt(zcu) and bits != wasm_bits) { + _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits)); + } + try cg.addTag(.i64_popcnt); + try cg.addTag(.i32_wrap_i64); + try cg.emitWValue(operand); + }, + 128 => { + _ = try cg.load(operand, Type.u64, 0); + try cg.addTag(.i64_popcnt); + _ = try cg.load(operand, Type.u64, 8); + if (op_ty.isSignedInt(zcu) and bits != wasm_bits) { + _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits - 64)); + } + try cg.addTag(.i64_popcnt); + try cg.addTag(.i64_add); + try cg.addTag(.i32_wrap_i64); + }, + else => unreachable, + } + + return cg.finishAir(inst, .stack, &.{ty_op.operand}); +} + +fn airBitReverse(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const ty = cg.typeOf(ty_op.operand); + + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: Implement @bitReverse for vectors", .{}); + } + + const int_info = ty.intInfo(zcu); + const bits = int_info.bits; + const wasm_bits = toWasmBits(bits) orelse { + return cg.fail("TODO: Implement @bitReverse for integers with bitsize '{d}'", .{bits}); + }; + + switch (wasm_bits) { + 32 => { + const intrin_ret = try cg.callIntrinsic( + .__bitreversesi2, + &.{.u32_type}, + Type.u32, + &.{operand}, + ); + const result = if (bits == 32) + intrin_ret + else + try cg.binOp(intrin_ret, .{ .imm32 = 32 - bits }, ty, .shr); + return cg.finishAir(inst, result, &.{ty_op.operand}); + }, + 64 => { + const intrin_ret = try cg.callIntrinsic( + .__bitreversedi2, + &.{.u64_type}, + Type.u64, + &.{operand}, + ); + const result = if (bits == 64) + intrin_ret + else + try cg.binOp(intrin_ret, .{ .imm64 = 64 - bits }, ty, .shr); + return cg.finishAir(inst, result, &.{ty_op.operand}); + }, + 128 => { + const result = try cg.allocStack(ty); + + try cg.emitWValue(result); + const first_half = try cg.load(operand, Type.u64, 8); + const intrin_ret_first = try cg.callIntrinsic( + .__bitreversedi2, + &.{.u64_type}, + Type.u64, + &.{first_half}, + ); + try cg.emitWValue(intrin_ret_first); + if (bits < 128) { + try cg.emitWValue(.{ .imm64 = 128 - bits }); + try cg.addTag(.i64_shr_u); + } + try cg.emitWValue(result); + const second_half = try cg.load(operand, Type.u64, 0); + const intrin_ret_second = try cg.callIntrinsic( + .__bitreversedi2, + &.{.u64_type}, + Type.u64, + &.{second_half}, + ); + try cg.emitWValue(intrin_ret_second); + if (bits == 128) { + try cg.store(.stack, .stack, Type.u64, result.offset() + 8); + try cg.store(.stack, .stack, Type.u64, result.offset()); + } else { + var tmp = try cg.allocLocal(Type.u64); + defer tmp.free(cg); + try cg.addLocal(.local_tee, tmp.local.value); + try cg.emitWValue(.{ .imm64 = 128 - bits }); + if (ty.isSignedInt(zcu)) { + try cg.addTag(.i64_shr_s); + } else { + try cg.addTag(.i64_shr_u); + } + try cg.store(.stack, .stack, Type.u64, result.offset() + 8); + try cg.addLocal(.local_get, tmp.local.value); + try cg.emitWValue(.{ .imm64 = bits - 64 }); + try cg.addTag(.i64_shl); + try cg.addTag(.i64_or); + try cg.store(.stack, .stack, Type.u64, result.offset()); + } + return cg.finishAir(inst, result, &.{ty_op.operand}); + }, + else => unreachable, + } +} + +fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + // Each entry to this table is a slice (ptr+len). + // The operand in this instruction represents the index within this table. + // This means to get the final name, we emit the base pointer and then perform + // pointer arithmetic to find the pointer to this slice and return that. + // + // As the names are global and the slice elements are constant, we do not have + // to make a copy of the ptr+value but can point towards them directly. + const pt = cg.pt; + const name_ty = Type.slice_const_u8_sentinel_0; + const abi_size = name_ty.abiSize(pt.zcu); + + // Lowers to a i32.const or i64.const with the error table memory address. + cg.error_name_table_ref_count += 1; + try cg.addTag(.error_name_table_ref); + try cg.emitWValue(operand); + switch (cg.ptr_size) { + .wasm32 => { + try cg.addImm32(@intCast(abi_size)); + try cg.addTag(.i32_mul); + try cg.addTag(.i32_add); + }, + .wasm64 => { + try cg.addImm64(abi_size); + try cg.addTag(.i64_mul); + try cg.addTag(.i64_add); + }, + } + + return cg.finishAir(inst, .stack, &.{un_op}); +} + +fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void { + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const slice_ptr = try cg.resolveInst(ty_op.operand); + const result = try cg.buildPointerOffset(slice_ptr, offset, .new); + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +/// NOTE: Allocates place for result on virtual stack, when integer size > 64 bits +fn intZeroValue(cg: *CodeGen, ty: Type) InnerError!WValue { + const zcu = cg.pt.zcu; + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: Implement intZeroValue for integer bitsize: {d}", .{int_info.bits}); + }; + switch (wasm_bits) { + 32 => return .{ .imm32 = 0 }, + 64 => return .{ .imm64 = 0 }, + 128 => { + const result = try cg.allocStack(ty); + try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0); + try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8); + return result; + }, + else => unreachable, + } +} + +fn airAddSubWithOverflow(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { + assert(op == .add or op == .sub); + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const lhs = try cg.resolveInst(extra.lhs); + const rhs = try cg.resolveInst(extra.rhs); + const ty = cg.typeOf(extra.lhs); + const pt = cg.pt; + const zcu = pt.zcu; + + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: Implement overflow arithmetic for vectors", .{}); + } + + const int_info = ty.intInfo(zcu); + const is_signed = int_info.signedness == .signed; + if (int_info.bits > 128) { + return cg.fail("TODO: Implement {{add/sub}}_with_overflow for integer bitsize: {d}", .{int_info.bits}); + } + + const op_result = try cg.wrapBinOp(lhs, rhs, ty, op); + var op_tmp = try op_result.toLocal(cg, ty); + defer op_tmp.free(cg); + + const cmp_op: std.math.CompareOperator = switch (op) { + .add => .lt, + .sub => .gt, + else => unreachable, + }; + const overflow_bit = if (is_signed) blk: { + const zero = try intZeroValue(cg, ty); + const rhs_is_neg = try cg.cmp(rhs, zero, ty, .lt); + const overflow_cmp = try cg.cmp(op_tmp, lhs, ty, cmp_op); + break :blk try cg.cmp(rhs_is_neg, overflow_cmp, Type.u1, .neq); + } else try cg.cmp(op_tmp, lhs, ty, cmp_op); + var bit_tmp = try overflow_bit.toLocal(cg, Type.u1); + defer bit_tmp.free(cg); + + const result = try cg.allocStack(cg.typeOfIndex(inst)); + const offset: u32 = @intCast(ty.abiSize(zcu)); + try cg.store(result, op_tmp, ty, 0); + try cg.store(result, bit_tmp, Type.u1, offset); + + return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); +} + +fn airShlWithOverflow(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const lhs = try cg.resolveInst(extra.lhs); + const rhs = try cg.resolveInst(extra.rhs); + const ty = cg.typeOf(extra.lhs); + const rhs_ty = cg.typeOf(extra.rhs); + + if (ty.isVector(zcu)) { + if (!rhs_ty.isVector(zcu)) { + return cg.fail("TODO: implement vector 'shl_with_overflow' with scalar rhs", .{}); + } else { + return cg.fail("TODO: implement vector 'shl_with_overflow'", .{}); + } + } + + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: implement 'shl_with_overflow' for integer bitsize: {d}", .{int_info.bits}); + }; + + // Ensure rhs is coerced to lhs as they must have the same WebAssembly types + // before we can perform any binary operation. + const rhs_wasm_bits = toWasmBits(rhs_ty.intInfo(zcu).bits).?; + // If wasm_bits == 128, compiler-rt expects i32 for shift + const rhs_final = if (wasm_bits != rhs_wasm_bits and wasm_bits == 64) blk: { + const rhs_casted = try cg.intcast(rhs, rhs_ty, ty); + break :blk try rhs_casted.toLocal(cg, ty); + } else rhs; + + var shl = try (try cg.wrapBinOp(lhs, rhs_final, ty, .shl)).toLocal(cg, ty); + defer shl.free(cg); + + const overflow_bit = blk: { + const shr = try cg.binOp(shl, rhs_final, ty, .shr); + break :blk try cg.cmp(shr, lhs, ty, .neq); + }; + var overflow_local = try overflow_bit.toLocal(cg, Type.u1); + defer overflow_local.free(cg); + + const result = try cg.allocStack(cg.typeOfIndex(inst)); + const offset: u32 = @intCast(ty.abiSize(zcu)); + try cg.store(result, shl, ty, 0); + try cg.store(result, overflow_local, Type.u1, offset); + + return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); +} + +fn airMulWithOverflow(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data; + + const lhs = try cg.resolveInst(extra.lhs); + const rhs = try cg.resolveInst(extra.rhs); + const ty = cg.typeOf(extra.lhs); + const pt = cg.pt; + const zcu = pt.zcu; + + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: Implement overflow arithmetic for vectors", .{}); + } + + // We store the bit if it's overflowed or not in this. As it's zero-initialized + // we only need to update it if an overflow (or underflow) occurred. + var overflow_bit = try cg.ensureAllocLocal(Type.u1); + defer overflow_bit.free(cg); + + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: Implement `@mulWithOverflow` for integer bitsize: {d}", .{int_info.bits}); + }; + + const zero: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = 0 }, + 64, 128 => .{ .imm64 = 0 }, + else => unreachable, + }; + + // for 32 bit integers we upcast it to a 64bit integer + const mul = if (wasm_bits == 32) blk: { + const new_ty = if (int_info.signedness == .signed) Type.i64 else Type.u64; + const lhs_upcast = try cg.intcast(lhs, ty, new_ty); + const rhs_upcast = try cg.intcast(rhs, ty, new_ty); + const bin_op = try (try cg.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(cg, new_ty); + const res = try (try cg.trunc(bin_op, ty, new_ty)).toLocal(cg, ty); + const res_upcast = try cg.intcast(res, ty, new_ty); + _ = try cg.cmp(res_upcast, bin_op, new_ty, .neq); + try cg.addLocal(.local_set, overflow_bit.local.value); + break :blk res; + } else if (wasm_bits == 64) blk: { + const new_ty = if (int_info.signedness == .signed) Type.i128 else Type.u128; + const lhs_upcast = try cg.intcast(lhs, ty, new_ty); + const rhs_upcast = try cg.intcast(rhs, ty, new_ty); + const bin_op = try (try cg.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(cg, new_ty); + const res = try (try cg.trunc(bin_op, ty, new_ty)).toLocal(cg, ty); + const res_upcast = try cg.intcast(res, ty, new_ty); + _ = try cg.cmp(res_upcast, bin_op, new_ty, .neq); + try cg.addLocal(.local_set, overflow_bit.local.value); + break :blk res; + } else if (int_info.bits == 128 and int_info.signedness == .unsigned) blk: { + var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64); + defer lhs_lsb.free(cg); + var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64); + defer lhs_msb.free(cg); + var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64); + defer rhs_lsb.free(cg); + var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64); + defer rhs_msb.free(cg); + + const cross_1 = try cg.callIntrinsic( + .__multi3, + &[_]InternPool.Index{.i64_type} ** 4, + Type.i128, + &.{ lhs_msb, zero, rhs_lsb, zero }, + ); + const cross_2 = try cg.callIntrinsic( + .__multi3, + &[_]InternPool.Index{.i64_type} ** 4, + Type.i128, + &.{ rhs_msb, zero, lhs_lsb, zero }, + ); + const mul_lsb = try cg.callIntrinsic( + .__multi3, + &[_]InternPool.Index{.i64_type} ** 4, + Type.i128, + &.{ rhs_lsb, zero, lhs_lsb, zero }, + ); + + const rhs_msb_not_zero = try cg.cmp(rhs_msb, zero, Type.u64, .neq); + const lhs_msb_not_zero = try cg.cmp(lhs_msb, zero, Type.u64, .neq); + const both_msb_not_zero = try cg.binOp(rhs_msb_not_zero, lhs_msb_not_zero, Type.bool, .@"and"); + const cross_1_msb = try cg.load(cross_1, Type.u64, 8); + const cross_1_msb_not_zero = try cg.cmp(cross_1_msb, zero, Type.u64, .neq); + const cond_1 = try cg.binOp(both_msb_not_zero, cross_1_msb_not_zero, Type.bool, .@"or"); + const cross_2_msb = try cg.load(cross_2, Type.u64, 8); + const cross_2_msb_not_zero = try cg.cmp(cross_2_msb, zero, Type.u64, .neq); + const cond_2 = try cg.binOp(cond_1, cross_2_msb_not_zero, Type.bool, .@"or"); + + const cross_1_lsb = try cg.load(cross_1, Type.u64, 0); + const cross_2_lsb = try cg.load(cross_2, Type.u64, 0); + const cross_add = try cg.binOp(cross_1_lsb, cross_2_lsb, Type.u64, .add); + + var mul_lsb_msb = try (try cg.load(mul_lsb, Type.u64, 8)).toLocal(cg, Type.u64); + defer mul_lsb_msb.free(cg); + var all_add = try (try cg.binOp(cross_add, mul_lsb_msb, Type.u64, .add)).toLocal(cg, Type.u64); + defer all_add.free(cg); + const add_overflow = try cg.cmp(all_add, mul_lsb_msb, Type.u64, .lt); + + // result for overflow bit + _ = try cg.binOp(cond_2, add_overflow, Type.bool, .@"or"); + try cg.addLocal(.local_set, overflow_bit.local.value); + + const tmp_result = try cg.allocStack(Type.u128); + try cg.emitWValue(tmp_result); + const mul_lsb_lsb = try cg.load(mul_lsb, Type.u64, 0); + try cg.store(.stack, mul_lsb_lsb, Type.u64, tmp_result.offset()); + try cg.store(tmp_result, all_add, Type.u64, 8); + break :blk tmp_result; + } else if (int_info.bits == 128 and int_info.signedness == .signed) blk: { + const overflow_ret = try cg.allocStack(Type.i32); + const res = try cg.callIntrinsic( + .__muloti4, + &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type }, + Type.i128, + &.{ lhs, rhs, overflow_ret }, + ); + _ = try cg.load(overflow_ret, Type.i32, 0); + try cg.addLocal(.local_set, overflow_bit.local.value); + break :blk res; + } else return cg.fail("TODO: @mulWithOverflow for {f}", .{ty.fmt(pt)}); + var bin_op_local = try mul.toLocal(cg, ty); + defer bin_op_local.free(cg); + + const result = try cg.allocStack(cg.typeOfIndex(inst)); + const offset: u32 = @intCast(ty.abiSize(zcu)); + try cg.store(result, bin_op_local, ty, 0); + try cg.store(result, overflow_bit, Type.u1, offset); + + return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); +} + +fn airMaxMin( + cg: *CodeGen, + inst: Air.Inst.Index, + op: enum { fmax, fmin }, + cmp_op: std.math.CompareOperator, +) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ty = cg.typeOfIndex(inst); + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: `@maximum` and `@minimum` for vectors", .{}); + } + + if (ty.abiSize(zcu) > 16) { + return cg.fail("TODO: `@maximum` and `@minimum` for types larger than 16 bytes", .{}); + } + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + if (ty.zigTypeTag(zcu) == .float) { + const intrinsic = switch (op) { + inline .fmin, .fmax => |ct_op| switch (ty.floatBits(cg.target)) { + inline 16, 32, 64, 80, 128 => |bits| @field( + Mir.Intrinsic, + libcFloatPrefix(bits) ++ @tagName(ct_op) ++ libcFloatSuffix(bits), + ), + else => unreachable, + }, + }; + const result = try cg.callIntrinsic(intrinsic, &.{ ty.ip_index, ty.ip_index }, ty, &.{ lhs, rhs }); + try cg.lowerToStack(result); + } else { + // operands to select from + try cg.lowerToStack(lhs); + try cg.lowerToStack(rhs); + _ = try cg.cmp(lhs, rhs, ty, cmp_op); + + // based on the result from comparison, return operand 0 or 1. + try cg.addTag(.select); + } + + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airMulAdd(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data; + + const ty = cg.typeOfIndex(inst); + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: `@mulAdd` for vectors", .{}); + } + + const addend = try cg.resolveInst(pl_op.operand); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + const result = if (ty.floatBits(cg.target) == 16) fl_result: { + const rhs_ext = try cg.fpext(rhs, ty, Type.f32); + const lhs_ext = try cg.fpext(lhs, ty, Type.f32); + const addend_ext = try cg.fpext(addend, ty, Type.f32); + // call to compiler-rt `fn fmaf(f32, f32, f32) f32` + const result = try cg.callIntrinsic( + .fmaf, + &.{ .f32_type, .f32_type, .f32_type }, + Type.f32, + &.{ rhs_ext, lhs_ext, addend_ext }, + ); + break :fl_result try cg.fptrunc(result, Type.f32, ty); + } else result: { + const mul_result = try cg.binOp(lhs, rhs, ty, .mul); + break :result try cg.binOp(mul_result, addend, ty, .add); + }; + + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand }); +} + +fn airClz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const ty = cg.typeOf(ty_op.operand); + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: `@clz` for vectors", .{}); + } + + const operand = try cg.resolveInst(ty_op.operand); + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: `@clz` for integers with bitsize '{d}'", .{int_info.bits}); + }; + + switch (wasm_bits) { + 32 => { + if (int_info.signedness == .signed) { + const mask = ~@as(u32, 0) >> @intCast(32 - int_info.bits); + _ = try cg.binOp(operand, .{ .imm32 = mask }, ty, .@"and"); + } else { + try cg.emitWValue(operand); + } + try cg.addTag(.i32_clz); + }, + 64 => { + if (int_info.signedness == .signed) { + const mask = ~@as(u64, 0) >> @intCast(64 - int_info.bits); + _ = try cg.binOp(operand, .{ .imm64 = mask }, ty, .@"and"); + } else { + try cg.emitWValue(operand); + } + try cg.addTag(.i64_clz); + try cg.addTag(.i32_wrap_i64); + }, + 128 => { + var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64); + defer msb.free(cg); + + try cg.emitWValue(msb); + try cg.addTag(.i64_clz); + _ = try cg.load(operand, Type.u64, 0); + try cg.addTag(.i64_clz); + try cg.emitWValue(.{ .imm64 = 64 }); + try cg.addTag(.i64_add); + _ = try cg.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq); + try cg.addTag(.select); + try cg.addTag(.i32_wrap_i64); + }, + else => unreachable, + } + + if (wasm_bits != int_info.bits) { + try cg.emitWValue(.{ .imm32 = wasm_bits - int_info.bits }); + try cg.addTag(.i32_sub); + } + + return cg.finishAir(inst, .stack, &.{ty_op.operand}); +} + +fn airCtz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const ty = cg.typeOf(ty_op.operand); + + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: `@ctz` for vectors", .{}); + } + + const operand = try cg.resolveInst(ty_op.operand); + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: `@clz` for integers with bitsize '{d}'", .{int_info.bits}); + }; + + switch (wasm_bits) { + 32 => { + if (wasm_bits != int_info.bits) { + const val: u32 = @as(u32, 1) << @as(u5, @intCast(int_info.bits)); + // leave value on the stack + _ = try cg.binOp(operand, .{ .imm32 = val }, ty, .@"or"); + } else try cg.emitWValue(operand); + try cg.addTag(.i32_ctz); + }, + 64 => { + if (wasm_bits != int_info.bits) { + const val: u64 = @as(u64, 1) << @as(u6, @intCast(int_info.bits)); + // leave value on the stack + _ = try cg.binOp(operand, .{ .imm64 = val }, ty, .@"or"); + } else try cg.emitWValue(operand); + try cg.addTag(.i64_ctz); + try cg.addTag(.i32_wrap_i64); + }, + 128 => { + var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64); + defer lsb.free(cg); + + try cg.emitWValue(lsb); + try cg.addTag(.i64_ctz); + _ = try cg.load(operand, Type.u64, 8); + if (wasm_bits != int_info.bits) { + try cg.addImm64(@as(u64, 1) << @as(u6, @intCast(int_info.bits - 64))); + try cg.addTag(.i64_or); + } + try cg.addTag(.i64_ctz); + try cg.addImm64(64); + if (wasm_bits != int_info.bits) { + try cg.addTag(.i64_or); + } else { + try cg.addTag(.i64_add); + } + _ = try cg.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq); + try cg.addTag(.select); + try cg.addTag(.i32_wrap_i64); + }, + else => unreachable, + } + + return cg.finishAir(inst, .stack, &.{ty_op.operand}); +} + +fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const dbg_stmt = cg.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; + try cg.addInst(.{ .tag = .dbg_line, .data = .{ + .payload = try cg.addExtra(Mir.DbgLineColumn{ + .line = dbg_stmt.line, + .column = dbg_stmt.column, + }), + } }); + return cg.finishAir(inst, .none, &.{}); +} + +fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + // TODO + try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); +} + +fn airDbgVar( + cg: *CodeGen, + inst: Air.Inst.Index, + local_tag: link.File.Dwarf.WipNav.LocalVarTag, + is_ptr: bool, +) InnerError!void { + _ = is_ptr; + _ = local_tag; + return cg.finishAir(inst, .none, &.{}); +} + +fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const err_union = try cg.resolveInst(pl_op.operand); + const extra = cg.air.extraData(Air.Try, pl_op.payload); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); + const err_union_ty = cg.typeOf(pl_op.operand); + const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false); + return cg.finishAir(inst, result, &.{pl_op.operand}); +} + +fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload); + const err_union_ptr = try cg.resolveInst(extra.data.ptr); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); + const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu); + const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true); + return cg.finishAir(inst, result, &.{extra.data.ptr}); +} + +fn lowerTry( + cg: *CodeGen, + inst: Air.Inst.Index, + err_union: WValue, + body: []const Air.Inst.Index, + err_union_ty: Type, + operand_is_ptr: bool, +) InnerError!WValue { + const zcu = cg.pt.zcu; + + const pl_ty = err_union_ty.errorUnionPayload(zcu); + const pl_has_bits = pl_ty.hasRuntimeBitsIgnoreComptime(zcu); + + if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { + // Block we can jump out of when error is not set + try cg.startBlock(.block, .empty); + + // check if the error tag is set for the error union. + try cg.emitWValue(err_union); + if (pl_has_bits or operand_is_ptr) { + const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + try cg.addMemArg(.i32_load16_u, .{ + .offset = err_union.offset() + err_offset, + .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?), + }); + } + try cg.addTag(.i32_eqz); + try cg.addLabel(.br_if, 0); // jump out of block when error is '0' + + const liveness = cg.liveness.getCondBr(inst); + try cg.branches.append(cg.gpa, .{}); + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.else_deaths.len + liveness.then_deaths.len); + defer { + var branch = cg.branches.pop().?; + branch.deinit(cg.gpa); + } + try cg.genBody(body); + try cg.endBlock(); + } + + // if we reach here it means error was not set, and we want the payload + if (!pl_has_bits and !operand_is_ptr) { + return .none; + } + + const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); + if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) { + return buildPointerOffset(cg, err_union, pl_offset, .new); + } + const payload = try cg.load(err_union, pl_ty, pl_offset); + return payload.toLocal(cg, pl_ty); +} + +fn airByteSwap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const ty = cg.typeOfIndex(inst); + const operand = try cg.resolveInst(ty_op.operand); + + if (ty.zigTypeTag(zcu) == .vector) { + return cg.fail("TODO: @byteSwap for vectors", .{}); + } + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}); + }; + + // bytes are no-op + if (int_info.bits == 8) { + return cg.finishAir(inst, cg.reuseOperand(ty_op.operand, operand), &.{ty_op.operand}); + } + + const result = result: { + switch (wasm_bits) { + 32 => { + const intrin_ret = try cg.callIntrinsic( + .__bswapsi2, + &.{.u32_type}, + Type.u32, + &.{operand}, + ); + break :result if (int_info.bits == 32) + intrin_ret + else + try cg.binOp(intrin_ret, .{ .imm32 = 32 - int_info.bits }, ty, .shr); + }, + 64 => { + const intrin_ret = try cg.callIntrinsic( + .__bswapdi2, + &.{.u64_type}, + Type.u64, + &.{operand}, + ); + break :result if (int_info.bits == 64) + intrin_ret + else + try cg.binOp(intrin_ret, .{ .imm64 = 64 - int_info.bits }, ty, .shr); + }, + else => return cg.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}), + } + }; + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +fn airDiv(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ty = cg.typeOfIndex(inst); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + const result = try cg.binOp(lhs, rhs, ty, .div); + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airDivTrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ty = cg.typeOfIndex(inst); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + const div_result = try cg.binOp(lhs, rhs, ty, .div); + + if (ty.isAnyFloat()) { + const trunc_result = try cg.floatOp(.trunc, ty, &.{div_result}); + return cg.finishAir(inst, trunc_result, &.{ bin_op.lhs, bin_op.rhs }); + } + + return cg.finishAir(inst, div_result, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airDivFloor(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const zcu = cg.pt.zcu; + const ty = cg.typeOfIndex(inst); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + if (ty.isUnsignedInt(zcu)) { + _ = try cg.binOp(lhs, rhs, ty, .div); + } else if (ty.isSignedInt(zcu)) { + const int_bits = ty.intInfo(zcu).bits; + const wasm_bits = toWasmBits(int_bits) orelse { + return cg.fail("TODO: `@divFloor` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); + }; + + if (wasm_bits > 64) { + return cg.fail("TODO: `@divFloor` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); + } + + const zero: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = 0 }, + 64 => .{ .imm64 = 0 }, + else => unreachable, + }; + + // tee leaves the value on the stack and stores it in a local. + const quotient = try cg.allocLocal(ty); + _ = try cg.binOp(lhs, rhs, ty, .div); + try cg.addLocal(.local_tee, quotient.local.value); + + // select takes a 32 bit value as the condition, so in the 64 bit case we use eqz to narrow + // the 64 bit value we want to use as the condition to 32 bits. + // This also inverts the condition (non 0 => 0, 0 => 1), so we put the adjusted and + // non-adjusted quotients on the stack in the opposite order for 32 vs 64 bits. + if (wasm_bits == 64) { + try cg.emitWValue(quotient); + } + + // 0 if the signs of rhs_wasm and lhs_wasm are the same, 1 otherwise. + _ = try cg.binOp(lhs, rhs, ty, .xor); + _ = try cg.cmp(.stack, zero, ty, .lt); + + switch (wasm_bits) { + 32 => { + try cg.addTag(.i32_sub); + try cg.emitWValue(quotient); + }, + 64 => { + try cg.addTag(.i64_extend_i32_u); + try cg.addTag(.i64_sub); + }, + else => unreachable, + } + + _ = try cg.binOp(lhs, rhs, ty, .rem); + + if (wasm_bits == 64) { + try cg.addTag(.i64_eqz); + } + + try cg.addTag(.select); + + // We need to zero the high bits because N bit comparisons consider all 32 or 64 bits, and + // expect all but the lowest N bits to be 0. + // TODO: Should we be zeroing the high bits here or should we be ignoring the high bits + // when performing comparisons? + if (int_bits != wasm_bits) { + _ = try cg.wrapOperand(.stack, ty); + } + } else { + const float_bits = ty.floatBits(cg.target); + if (float_bits > 64) { + return cg.fail("TODO: `@divFloor` for floats with bitsize: {d}", .{float_bits}); + } + const is_f16 = float_bits == 16; + + const lhs_wasm = if (is_f16) try cg.fpext(lhs, Type.f16, Type.f32) else lhs; + const rhs_wasm = if (is_f16) try cg.fpext(rhs, Type.f16, Type.f32) else rhs; + + try cg.emitWValue(lhs_wasm); + try cg.emitWValue(rhs_wasm); + + switch (float_bits) { + 16, 32 => { + try cg.addTag(.f32_div); + try cg.addTag(.f32_floor); + }, + 64 => { + try cg.addTag(.f64_div); + try cg.addTag(.f64_floor); + }, + else => unreachable, + } + + if (is_f16) { + _ = try cg.fptrunc(.stack, Type.f32, Type.f16); + } + } + + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airRem(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ty = cg.typeOfIndex(inst); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + const result = try cg.binOp(lhs, rhs, ty, .rem); + + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +/// Remainder after floor division, defined by: +/// @divFloor(a, b) * b + @mod(a, b) = a +fn airMod(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const pt = cg.pt; + const zcu = pt.zcu; + const ty = cg.typeOfIndex(inst); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + const result = result: { + if (ty.isUnsignedInt(zcu)) { + break :result try cg.binOp(lhs, rhs, ty, .rem); + } + if (ty.isSignedInt(zcu)) { + // The wasm rem instruction gives the remainder after truncating division (rounding towards + // 0), equivalent to @rem. + // We make use of the fact that: + // @mod(a, b) = @rem(@rem(a, b) + b, b) + const int_bits = ty.intInfo(zcu).bits; + const wasm_bits = toWasmBits(int_bits) orelse { + return cg.fail("TODO: `@mod` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); + }; + + if (wasm_bits > 64) { + return cg.fail("TODO: `@mod` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits}); + } + + _ = try cg.binOp(lhs, rhs, ty, .rem); + _ = try cg.binOp(.stack, rhs, ty, .add); + break :result try cg.binOp(.stack, rhs, ty, .rem); + } + if (ty.isAnyFloat()) { + const rem = try cg.binOp(lhs, rhs, ty, .rem); + const add = try cg.binOp(rem, rhs, ty, .add); + break :result try cg.binOp(add, rhs, ty, .rem); + } + return cg.fail("TODO: @mod for {f}", .{ty.fmt(pt)}); + }; + + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airSatMul(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const pt = cg.pt; + const zcu = pt.zcu; + const ty = cg.typeOfIndex(inst); + const int_info = ty.intInfo(zcu); + const is_signed = int_info.signedness == .signed; + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const wasm_bits = toWasmBits(int_info.bits) orelse { + return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)}); + }; + + switch (wasm_bits) { + 32 => { + const upcast_ty: Type = if (is_signed) Type.i64 else Type.u64; + const lhs_up = try cg.intcast(lhs, ty, upcast_ty); + const rhs_up = try cg.intcast(rhs, ty, upcast_ty); + var mul_res = try (try cg.binOp(lhs_up, rhs_up, upcast_ty, .mul)).toLocal(cg, upcast_ty); + defer mul_res.free(cg); + if (is_signed) { + const imm_max: WValue = .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - (int_info.bits - 1)) }; + try cg.emitWValue(mul_res); + try cg.emitWValue(imm_max); + _ = try cg.cmp(mul_res, imm_max, upcast_ty, .lt); + try cg.addTag(.select); + + var tmp = try cg.allocLocal(upcast_ty); + defer tmp.free(cg); + try cg.addLocal(.local_set, tmp.local.value); + + const imm_min: WValue = .{ .imm64 = ~@as(u64, 0) << @intCast(int_info.bits - 1) }; + try cg.emitWValue(tmp); + try cg.emitWValue(imm_min); + _ = try cg.cmp(tmp, imm_min, upcast_ty, .gt); + try cg.addTag(.select); + } else { + const imm_max: WValue = .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_info.bits) }; + try cg.emitWValue(mul_res); + try cg.emitWValue(imm_max); + _ = try cg.cmp(mul_res, imm_max, upcast_ty, .lt); + try cg.addTag(.select); + } + try cg.addTag(.i32_wrap_i64); + }, + 64 => { + if (!(int_info.bits == 64 and int_info.signedness == .signed)) { + return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)}); + } + const overflow_ret = try cg.allocStack(Type.i32); + _ = try cg.callIntrinsic( + .__mulodi4, + &[_]InternPool.Index{ .i64_type, .i64_type, .usize_type }, + Type.i64, + &.{ lhs, rhs, overflow_ret }, + ); + const xor = try cg.binOp(lhs, rhs, Type.i64, .xor); + const sign_v = try cg.binOp(xor, .{ .imm64 = 63 }, Type.i64, .shr); + _ = try cg.binOp(sign_v, .{ .imm64 = ~@as(u63, 0) }, Type.i64, .xor); + _ = try cg.load(overflow_ret, Type.i32, 0); + try cg.addTag(.i32_eqz); + try cg.addTag(.select); + }, + 128 => { + if (!(int_info.bits == 128 and int_info.signedness == .signed)) { + return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)}); + } + const overflow_ret = try cg.allocStack(Type.i32); + const ret = try cg.callIntrinsic( + .__muloti4, + &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type }, + Type.i128, + &.{ lhs, rhs, overflow_ret }, + ); + try cg.lowerToStack(ret); + const xor = try cg.binOp(lhs, rhs, Type.i128, .xor); + const sign_v = try cg.binOp(xor, .{ .imm32 = 127 }, Type.i128, .shr); + + // xor ~@as(u127, 0) + try cg.emitWValue(sign_v); + const lsb = try cg.load(sign_v, Type.u64, 0); + _ = try cg.binOp(lsb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor); + try cg.store(.stack, .stack, Type.u64, sign_v.offset()); + try cg.emitWValue(sign_v); + const msb = try cg.load(sign_v, Type.u64, 8); + _ = try cg.binOp(msb, .{ .imm64 = ~@as(u63, 0) }, Type.u64, .xor); + try cg.store(.stack, .stack, Type.u64, sign_v.offset() + 8); + + try cg.lowerToStack(sign_v); + _ = try cg.load(overflow_ret, Type.i32, 0); + try cg.addTag(.i32_eqz); + try cg.addTag(.select); + }, + else => unreachable, + } + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airSatBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { + assert(op == .add or op == .sub); + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const zcu = cg.pt.zcu; + const ty = cg.typeOfIndex(inst); + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + + const int_info = ty.intInfo(zcu); + const is_signed = int_info.signedness == .signed; + + if (int_info.bits > 64) { + return cg.fail("TODO: saturating arithmetic for integers with bitsize '{d}'", .{int_info.bits}); + } + + if (is_signed) { + const result = try signedSat(cg, lhs, rhs, ty, op); + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); + } + + const wasm_bits = toWasmBits(int_info.bits).?; + var bin_result = try (try cg.binOp(lhs, rhs, ty, op)).toLocal(cg, ty); + defer bin_result.free(cg); + if (wasm_bits != int_info.bits and op == .add) { + const val: u64 = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(int_info.bits))) - 1)); + const imm_val: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = @intCast(val) }, + 64 => .{ .imm64 = val }, + else => unreachable, + }; + + try cg.emitWValue(bin_result); + try cg.emitWValue(imm_val); + _ = try cg.cmp(bin_result, imm_val, ty, .lt); + } else { + switch (wasm_bits) { + 32 => try cg.addImm32(if (op == .add) std.math.maxInt(u32) else 0), + 64 => try cg.addImm64(if (op == .add) std.math.maxInt(u64) else 0), + else => unreachable, + } + try cg.emitWValue(bin_result); + _ = try cg.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt); + } + + try cg.addTag(.select); + return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn signedSat(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue { + const pt = cg.pt; + const zcu = pt.zcu; + const int_info = ty.intInfo(zcu); + const wasm_bits = toWasmBits(int_info.bits).?; + const is_wasm_bits = wasm_bits == int_info.bits; + const ext_ty = if (!is_wasm_bits) try pt.intType(int_info.signedness, wasm_bits) else ty; + + const max_val: u64 = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(int_info.bits - 1))) - 1)); + const min_val: i64 = (-@as(i64, @intCast(@as(u63, @intCast(max_val))))) - 1; + const max_wvalue: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = @truncate(max_val) }, + 64 => .{ .imm64 = max_val }, + else => unreachable, + }; + const min_wvalue: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = @bitCast(@as(i32, @truncate(min_val))) }, + 64 => .{ .imm64 = @bitCast(min_val) }, + else => unreachable, + }; + + var bin_result = try (try cg.binOp(lhs, rhs, ext_ty, op)).toLocal(cg, ext_ty); + if (!is_wasm_bits) { + defer bin_result.free(cg); // not returned in this branch + try cg.emitWValue(bin_result); + try cg.emitWValue(max_wvalue); + _ = try cg.cmp(bin_result, max_wvalue, ext_ty, .lt); + try cg.addTag(.select); + try cg.addLocal(.local_set, bin_result.local.value); // re-use local + + try cg.emitWValue(bin_result); + try cg.emitWValue(min_wvalue); + _ = try cg.cmp(bin_result, min_wvalue, ext_ty, .gt); + try cg.addTag(.select); + try cg.addLocal(.local_set, bin_result.local.value); // re-use local + return (try cg.wrapOperand(bin_result, ty)).toLocal(cg, ty); + } else { + const zero: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = 0 }, + 64 => .{ .imm64 = 0 }, + else => unreachable, + }; + try cg.emitWValue(max_wvalue); + try cg.emitWValue(min_wvalue); + _ = try cg.cmp(bin_result, zero, ty, .lt); + try cg.addTag(.select); + try cg.emitWValue(bin_result); + // leave on stack + const cmp_zero_result = try cg.cmp(rhs, zero, ty, if (op == .add) .lt else .gt); + const cmp_bin_result = try cg.cmp(bin_result, lhs, ty, .lt); + _ = try cg.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor. + try cg.addTag(.select); + try cg.addLocal(.local_set, bin_result.local.value); // re-use local + return bin_result; + } +} + +fn airShlSat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const pt = cg.pt; + const zcu = pt.zcu; + + if (cg.typeOf(bin_op.lhs).isVector(zcu) and !cg.typeOf(bin_op.rhs).isVector(zcu)) { + return cg.fail("TODO: implement vector 'shl_sat' with scalar rhs", .{}); + } + + const ty = cg.typeOfIndex(inst); + const int_info = ty.intInfo(zcu); + const is_signed = int_info.signedness == .signed; + if (int_info.bits > 64) { + return cg.fail("TODO: Saturating shifting left for integers with bitsize '{d}'", .{int_info.bits}); + } + + const lhs = try cg.resolveInst(bin_op.lhs); + const rhs = try cg.resolveInst(bin_op.rhs); + const wasm_bits = toWasmBits(int_info.bits).?; + const result = try cg.allocLocal(ty); + + if (wasm_bits == int_info.bits) { + var shl = try (try cg.binOp(lhs, rhs, ty, .shl)).toLocal(cg, ty); + defer shl.free(cg); + var shr = try (try cg.binOp(shl, rhs, ty, .shr)).toLocal(cg, ty); + defer shr.free(cg); + + switch (wasm_bits) { + 32 => blk: { + if (!is_signed) { + try cg.addImm32(std.math.maxInt(u32)); + break :blk; + } + try cg.addImm32(@bitCast(@as(i32, std.math.minInt(i32)))); + try cg.addImm32(@bitCast(@as(i32, std.math.maxInt(i32)))); + _ = try cg.cmp(lhs, .{ .imm32 = 0 }, ty, .lt); + try cg.addTag(.select); + }, + 64 => blk: { + if (!is_signed) { + try cg.addImm64(std.math.maxInt(u64)); + break :blk; + } + try cg.addImm64(@bitCast(@as(i64, std.math.minInt(i64)))); + try cg.addImm64(@bitCast(@as(i64, std.math.maxInt(i64)))); + _ = try cg.cmp(lhs, .{ .imm64 = 0 }, ty, .lt); + try cg.addTag(.select); + }, + else => unreachable, + } + try cg.emitWValue(shl); + _ = try cg.cmp(lhs, shr, ty, .neq); + try cg.addTag(.select); + try cg.addLocal(.local_set, result.local.value); + } else { + const shift_size = wasm_bits - int_info.bits; + const shift_value: WValue = switch (wasm_bits) { + 32 => .{ .imm32 = shift_size }, + 64 => .{ .imm64 = shift_size }, + else => unreachable, + }; + const ext_ty = try pt.intType(int_info.signedness, wasm_bits); + + var shl_res = try (try cg.binOp(lhs, shift_value, ext_ty, .shl)).toLocal(cg, ext_ty); + defer shl_res.free(cg); + var shl = try (try cg.binOp(shl_res, rhs, ext_ty, .shl)).toLocal(cg, ext_ty); + defer shl.free(cg); + var shr = try (try cg.binOp(shl, rhs, ext_ty, .shr)).toLocal(cg, ext_ty); + defer shr.free(cg); + + switch (wasm_bits) { + 32 => blk: { + if (!is_signed) { + try cg.addImm32(std.math.maxInt(u32)); + break :blk; + } + + try cg.addImm32(@bitCast(@as(i32, std.math.minInt(i32)))); + try cg.addImm32(@bitCast(@as(i32, std.math.maxInt(i32)))); + _ = try cg.cmp(shl_res, .{ .imm32 = 0 }, ext_ty, .lt); + try cg.addTag(.select); + }, + 64 => blk: { + if (!is_signed) { + try cg.addImm64(std.math.maxInt(u64)); + break :blk; + } + + try cg.addImm64(@bitCast(@as(i64, std.math.minInt(i64)))); + try cg.addImm64(@bitCast(@as(i64, std.math.maxInt(i64)))); + _ = try cg.cmp(shl_res, .{ .imm64 = 0 }, ext_ty, .lt); + try cg.addTag(.select); + }, + else => unreachable, + } + try cg.emitWValue(shl); + _ = try cg.cmp(shl_res, shr, ext_ty, .neq); + try cg.addTag(.select); + try cg.addLocal(.local_set, result.local.value); + var shift_result = try cg.binOp(result, shift_value, ext_ty, .shr); + if (is_signed) { + shift_result = try cg.wrapOperand(shift_result, ty); + } + try cg.addLocal(.local_set, result.local.value); + } + + return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); +} + +/// Calls a compiler-rt intrinsic by creating an undefined symbol, +/// then lowering the arguments and calling the symbol as a function call. +/// This function call assumes the C-ABI. +/// Asserts arguments are not stack values when the return value is +/// passed as the first parameter. +/// May leave the return value on the stack. +fn callIntrinsic( + cg: *CodeGen, + intrinsic: Mir.Intrinsic, + param_types: []const InternPool.Index, + return_type: Type, + args: []const WValue, +) InnerError!WValue { + assert(param_types.len == args.len); + const zcu = cg.pt.zcu; + + // Always pass over C-ABI + + const want_sret_param = firstParamSRet(.{ .wasm_mvp = .{} }, return_type, zcu, cg.target); + // if we want return as first param, we allocate a pointer to stack, + // and emit it as our first argument + const sret = if (want_sret_param) blk: { + const sret_local = try cg.allocStack(return_type); + try cg.lowerToStack(sret_local); + break :blk sret_local; + } else .none; + + // Lower all arguments to the stack before we call our function + for (args, 0..) |arg, arg_i| { + assert(!(want_sret_param and arg == .stack)); + assert(Type.fromInterned(param_types[arg_i]).hasRuntimeBitsIgnoreComptime(zcu)); + try cg.lowerArg(.{ .wasm_mvp = .{} }, Type.fromInterned(param_types[arg_i]), arg); + } + + try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } }); + + if (!return_type.hasRuntimeBitsIgnoreComptime(zcu)) { + return .none; + } else if (return_type.isNoReturn(zcu)) { + try cg.addTag(.@"unreachable"); + return .none; + } else if (want_sret_param) { + return sret; + } else { + return .stack; + } +} + +fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try cg.resolveInst(un_op); + const enum_ty = cg.typeOf(un_op); + + const result_ptr = try cg.allocStack(cg.typeOfIndex(inst)); + try cg.lowerToStack(result_ptr); + try cg.emitWValue(operand); + try cg.addInst(.{ .tag = .call_tag_name, .data = .{ .ip_index = enum_ty.toIntern() } }); + + return cg.finishAir(inst, result_ptr, &.{un_op}); +} + +fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try cg.resolveInst(ty_op.operand); + const error_set_ty = ty_op.ty.toType(); + const result = try cg.allocLocal(Type.bool); + + const names = error_set_ty.errorSetNames(zcu); + var values = try std.array_list.Managed(u32).initCapacity(cg.gpa, names.len); + defer values.deinit(); + + var lowest: ?u32 = null; + var highest: ?u32 = null; + for (0..names.len) |name_index| { + const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?; + if (lowest) |*l| { + if (err_int < l.*) { + l.* = err_int; + } + } else { + lowest = err_int; + } + if (highest) |*h| { + if (err_int > h.*) { + highest = err_int; + } + } else { + highest = err_int; + } + + values.appendAssumeCapacity(err_int); + } + + // start block for 'true' branch + try cg.startBlock(.block, .empty); + // start block for 'false' branch + try cg.startBlock(.block, .empty); + // block for the jump table itself + try cg.startBlock(.block, .empty); + + // lower operand to determine jump table target + try cg.emitWValue(operand); + try cg.addImm32(lowest.?); + try cg.addTag(.i32_sub); + + // Account for default branch so always add '1' + const depth = @as(u32, @intCast(highest.? - lowest.? + 1)); + const jump_table: Mir.JumpTable = .{ .length = depth }; + const table_extra_index = try cg.addExtra(jump_table); + try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } }); + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, depth); + + var value: u32 = lowest.?; + while (value <= highest.?) : (value += 1) { + const idx: u32 = blk: { + for (values.items) |val| { + if (val == value) break :blk 1; + } + break :blk 0; + }; + cg.mir_extra.appendAssumeCapacity(idx); + } + try cg.endBlock(); + + // 'false' branch (i.e. error set does not have value + // ensure we set local to 0 in case the local was re-used. + try cg.addImm32(0); + try cg.addLocal(.local_set, result.local.value); + try cg.addLabel(.br, 1); + try cg.endBlock(); + + // 'true' branch + try cg.addImm32(1); + try cg.addLocal(.local_set, result.local.value); + try cg.addLabel(.br, 0); + try cg.endBlock(); + + return cg.finishAir(inst, result, &.{ty_op.operand}); +} + +inline fn useAtomicFeature(cg: *const CodeGen) bool { + return cg.target.cpu.has(.wasm, .atomics); +} + +fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + + const ptr_ty = cg.typeOf(extra.ptr); + const ty = ptr_ty.childType(zcu); + const result_ty = cg.typeOfIndex(inst); + + const ptr_operand = try cg.resolveInst(extra.ptr); + const expected_val = try cg.resolveInst(extra.expected_value); + const new_val = try cg.resolveInst(extra.new_value); + + const cmp_result = try cg.allocLocal(Type.bool); + + const ptr_val = if (cg.useAtomicFeature()) val: { + const val_local = try cg.allocLocal(ty); + try cg.emitWValue(ptr_operand); + try cg.lowerToStack(expected_val); + try cg.lowerToStack(new_val); + try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) { + 1 => .i32_atomic_rmw8_cmpxchg_u, + 2 => .i32_atomic_rmw16_cmpxchg_u, + 4 => .i32_atomic_rmw_cmpxchg, + 8 => .i32_atomic_rmw_cmpxchg, + else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), + }, .{ + .offset = ptr_operand.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }); + try cg.addLocal(.local_tee, val_local.local.value); + _ = try cg.cmp(.stack, expected_val, ty, .eq); + try cg.addLocal(.local_set, cmp_result.local.value); + break :val val_local; + } else val: { + if (ty.abiSize(zcu) > 8) { + return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{}); + } + const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty); + + try cg.lowerToStack(ptr_operand); + try cg.lowerToStack(new_val); + try cg.emitWValue(ptr_val); + _ = try cg.cmp(ptr_val, expected_val, ty, .eq); + try cg.addLocal(.local_tee, cmp_result.local.value); + try cg.addTag(.select); + try cg.store(.stack, .stack, ty, 0); + + break :val ptr_val; + }; + + const result = if (isByRef(result_ty, zcu, cg.target)) val: { + try cg.emitWValue(cmp_result); + try cg.addImm32(~@as(u32, 0)); + try cg.addTag(.i32_xor); + try cg.addImm32(1); + try cg.addTag(.i32_and); + const and_result = try WValue.toLocal(.stack, cg, Type.bool); + const result_ptr = try cg.allocStack(result_ty); + try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu)))); + try cg.store(result_ptr, ptr_val, ty, 0); + break :val result_ptr; + } else val: { + try cg.addImm32(0); + try cg.emitWValue(ptr_val); + try cg.emitWValue(cmp_result); + try cg.addTag(.select); + break :val .stack; + }; + + return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value }); +} + +fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const atomic_load = cg.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; + const ptr = try cg.resolveInst(atomic_load.ptr); + const ty = cg.typeOfIndex(inst); + + if (cg.useAtomicFeature()) { + const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { + 1 => .i32_atomic_load8_u, + 2 => .i32_atomic_load16_u, + 4 => .i32_atomic_load, + 8 => .i64_atomic_load, + else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), + }; + try cg.emitWValue(ptr); + try cg.addAtomicMemArg(tag, .{ + .offset = ptr.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }); + } else { + _ = try cg.load(ptr, ty, 0); + } + + return cg.finishAir(inst, .stack, &.{atomic_load.ptr}); +} + +fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = cg.air.extraData(Air.AtomicRmw, pl_op.payload).data; + + const ptr = try cg.resolveInst(pl_op.operand); + const operand = try cg.resolveInst(extra.operand); + const ty = cg.typeOfIndex(inst); + const op: std.builtin.AtomicRmwOp = extra.op(); + + if (cg.useAtomicFeature()) { + switch (op) { + .Max, + .Min, + .Nand, + => { + const tmp = try cg.load(ptr, ty, 0); + const value = try tmp.toLocal(cg, ty); + + // create a loop to cmpxchg the new value + try cg.startBlock(.loop, .empty); + + try cg.emitWValue(ptr); + try cg.emitWValue(value); + if (op == .Nand) { + const wasm_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?; + + const and_res = try cg.binOp(value, operand, ty, .@"and"); + if (wasm_bits == 32) + try cg.addImm32(~@as(u32, 0)) + else if (wasm_bits == 64) + try cg.addImm64(~@as(u64, 0)) + else + return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); + _ = try cg.binOp(and_res, .stack, ty, .xor); + } else { + try cg.emitWValue(value); + try cg.emitWValue(operand); + _ = try cg.cmp(value, operand, ty, if (op == .Max) .gt else .lt); + try cg.addTag(.select); + } + try cg.addAtomicMemArg( + switch (ty.abiSize(zcu)) { + 1 => .i32_atomic_rmw8_cmpxchg_u, + 2 => .i32_atomic_rmw16_cmpxchg_u, + 4 => .i32_atomic_rmw_cmpxchg, + 8 => .i64_atomic_rmw_cmpxchg, + else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}), + }, + .{ + .offset = ptr.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }, + ); + const select_res = try cg.allocLocal(ty); + try cg.addLocal(.local_tee, select_res.local.value); + _ = try cg.cmp(.stack, value, ty, .neq); // leave on stack so we can use it for br_if + + try cg.emitWValue(select_res); + try cg.addLocal(.local_set, value.local.value); + + try cg.addLabel(.br_if, 0); + try cg.endBlock(); + return cg.finishAir(inst, value, &.{ pl_op.operand, extra.operand }); + }, + + // the other operations have their own instructions for Wasm. + else => { + try cg.emitWValue(ptr); + try cg.emitWValue(operand); + const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { + 1 => switch (op) { + .Xchg => .i32_atomic_rmw8_xchg_u, + .Add => .i32_atomic_rmw8_add_u, + .Sub => .i32_atomic_rmw8_sub_u, + .And => .i32_atomic_rmw8_and_u, + .Or => .i32_atomic_rmw8_or_u, + .Xor => .i32_atomic_rmw8_xor_u, + else => unreachable, + }, + 2 => switch (op) { + .Xchg => .i32_atomic_rmw16_xchg_u, + .Add => .i32_atomic_rmw16_add_u, + .Sub => .i32_atomic_rmw16_sub_u, + .And => .i32_atomic_rmw16_and_u, + .Or => .i32_atomic_rmw16_or_u, + .Xor => .i32_atomic_rmw16_xor_u, + else => unreachable, + }, + 4 => switch (op) { + .Xchg => .i32_atomic_rmw_xchg, + .Add => .i32_atomic_rmw_add, + .Sub => .i32_atomic_rmw_sub, + .And => .i32_atomic_rmw_and, + .Or => .i32_atomic_rmw_or, + .Xor => .i32_atomic_rmw_xor, + else => unreachable, + }, + 8 => switch (op) { + .Xchg => .i64_atomic_rmw_xchg, + .Add => .i64_atomic_rmw_add, + .Sub => .i64_atomic_rmw_sub, + .And => .i64_atomic_rmw_and, + .Or => .i64_atomic_rmw_or, + .Xor => .i64_atomic_rmw_xor, + else => unreachable, + }, + else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}), + }; + try cg.addAtomicMemArg(tag, .{ + .offset = ptr.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }); + return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand }); + }, + } + } else { + const loaded = try cg.load(ptr, ty, 0); + const result = try loaded.toLocal(cg, ty); + + switch (op) { + .Xchg => { + try cg.store(ptr, operand, ty, 0); + }, + .Add, + .Sub, + .And, + .Or, + .Xor, + => { + try cg.emitWValue(ptr); + _ = try cg.binOp(result, operand, ty, switch (op) { + .Add => .add, + .Sub => .sub, + .And => .@"and", + .Or => .@"or", + .Xor => .xor, + else => unreachable, + }); + if (ty.isInt(zcu) and (op == .Add or op == .Sub)) { + _ = try cg.wrapOperand(.stack, ty); + } + try cg.store(.stack, .stack, ty, ptr.offset()); + }, + .Max, + .Min, + => { + try cg.emitWValue(ptr); + try cg.emitWValue(result); + try cg.emitWValue(operand); + _ = try cg.cmp(result, operand, ty, if (op == .Max) .gt else .lt); + try cg.addTag(.select); + try cg.store(.stack, .stack, ty, ptr.offset()); + }, + .Nand => { + const wasm_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?; + + try cg.emitWValue(ptr); + const and_res = try cg.binOp(result, operand, ty, .@"and"); + if (wasm_bits == 32) + try cg.addImm32(~@as(u32, 0)) + else if (wasm_bits == 64) + try cg.addImm64(~@as(u64, 0)) + else + return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); + _ = try cg.binOp(and_res, .stack, ty, .xor); + try cg.store(.stack, .stack, ty, ptr.offset()); + }, + } + + return cg.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); + } +} + +fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const zcu = cg.pt.zcu; + const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ptr = try cg.resolveInst(bin_op.lhs); + const operand = try cg.resolveInst(bin_op.rhs); + const ptr_ty = cg.typeOf(bin_op.lhs); + const ty = ptr_ty.childType(zcu); + + if (cg.useAtomicFeature()) { + const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { + 1 => .i32_atomic_store8, + 2 => .i32_atomic_store16, + 4 => .i32_atomic_store, + 8 => .i64_atomic_store, + else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), + }; + try cg.emitWValue(ptr); + try cg.lowerToStack(operand); + try cg.addAtomicMemArg(tag, .{ + .offset = ptr.offset(), + .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + }); + } else { + try cg.store(ptr, operand, ty, 0); + } + + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); +} + +fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + if (cg.initial_stack_value == .none) { + try cg.initializeStack(); + } + try cg.emitWValue(cg.bottom_stack_value); + return cg.finishAir(inst, .stack, &.{}); +} + +fn airRuntimeNavPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_nav = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; + const mod = cg.pt.zcu.navFileScope(cg.owner_nav).mod.?; + if (mod.single_threaded) { + const result: WValue = .{ .nav_ref = .{ + .nav_index = ty_nav.nav, + .offset = 0, + } }; + return cg.finishAir(inst, result, &.{}); + } + return cg.fail("TODO: thread-local variables", .{}); +} + +fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type { + const zcu = cg.pt.zcu; + return cg.air.typeOf(inst, &zcu.intern_pool); +} + +fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type { + const zcu = cg.pt.zcu; + return cg.air.typeOfIndex(inst, &zcu.intern_pool); +} + +fn floatCmpIntrinsic(op: std.math.CompareOperator, bits: u16) Mir.Intrinsic { + return switch (op) { + .lt => switch (bits) { + 80 => .__ltxf2, + 128 => .__lttf2, + else => unreachable, + }, + .lte => switch (bits) { + 80 => .__lexf2, + 128 => .__letf2, + else => unreachable, + }, + .eq => switch (bits) { + 80 => .__eqxf2, + 128 => .__eqtf2, + else => unreachable, + }, + .neq => switch (bits) { + 80 => .__nexf2, + 128 => .__netf2, + else => unreachable, + }, + .gte => switch (bits) { + 80 => .__gexf2, + 128 => .__getf2, + else => unreachable, + }, + .gt => switch (bits) { + 80 => .__gtxf2, + 128 => .__gttf2, + else => unreachable, + }, + }; +} diff --git a/src/codegen/wasm/Emit.zig b/src/codegen/wasm/Emit.zig new file mode 100644 index 0000000000000000000000000000000000000000..272d5519a6ff6c06f87a601fb58063e9fa3a5077 --- /dev/null +++ b/src/codegen/wasm/Emit.zig @@ -0,0 +1,996 @@ +const Emit = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; + +const Wasm = link.File.Wasm; +const Mir = @import("Mir.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); +const InternPool = @import("../../InternPool.zig"); +const codegen = @import("../../codegen.zig"); + +mir: Mir, +wasm: *Wasm, +/// The binary representation that will be emitted by this module. +code: *ArrayList(u8), + +pub const Error = error{ + OutOfMemory, +}; + +pub fn lowerToCode(emit: *Emit) Error!void { + const mir = &emit.mir; + const code = emit.code; + const wasm = emit.wasm; + const comp = wasm.base.comp; + const gpa = comp.gpa; + const is_obj = comp.config.output_mode == .Obj; + const target = &comp.root_mod.resolved_target.result; + const is_wasm32 = target.cpu.arch == .wasm32; + + const tags = mir.instructions.items(.tag); + const datas = mir.instructions.items(.data); + var inst: u32 = 0; + + loop: switch (tags[inst]) { + .dbg_epilogue_begin => { + return; + }, + .block, .loop => { + const block_type = datas[inst].block_type; + try code.ensureUnusedCapacity(gpa, 2); + code.appendAssumeCapacity(@intFromEnum(tags[inst])); + code.appendAssumeCapacity(@intFromEnum(block_type)); + + inst += 1; + continue :loop tags[inst]; + }, + .uav_ref => { + if (is_obj) { + try uavRefObj(wasm, code, datas[inst].ip_index, 0, is_wasm32); + } else { + try uavRefExe(wasm, code, datas[inst].ip_index, 0, is_wasm32); + } + inst += 1; + continue :loop tags[inst]; + }, + .uav_ref_off => { + const extra = mir.extraData(Mir.UavRefOff, datas[inst].payload).data; + if (is_obj) { + try uavRefObj(wasm, code, extra.value, extra.offset, is_wasm32); + } else { + try uavRefExe(wasm, code, extra.value, extra.offset, is_wasm32); + } + inst += 1; + continue :loop tags[inst]; + }, + .nav_ref => { + try navRefOff(wasm, code, .{ .nav_index = datas[inst].nav_index, .offset = 0 }, is_wasm32); + inst += 1; + continue :loop tags[inst]; + }, + .nav_ref_off => { + try navRefOff(wasm, code, mir.extraData(Mir.NavRefOff, datas[inst].payload).data, is_wasm32); + inst += 1; + continue :loop tags[inst]; + }, + .func_ref => { + const indirect_func_idx: Wasm.ZcuIndirectFunctionSetIndex = @enumFromInt( + wasm.zcu_indirect_function_set.getIndex(datas[inst].nav_index).?, + ); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i32_const)); + if (is_obj) { + @panic("TODO"); + } else { + writeUleb128(code, 1 + @intFromEnum(indirect_func_idx)); + } + inst += 1; + continue :loop tags[inst]; + }, + .dbg_line => { + inst += 1; + continue :loop tags[inst]; + }, + .errors_len => { + try code.ensureUnusedCapacity(gpa, 6); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i32_const)); + // MIR is lowered during flush, so there is indeed only one thread at this time. + const errors_len = 1 + comp.zcu.?.intern_pool.global_error_set.getNamesFromMainThread().len; + writeSleb128(code, errors_len); + + inst += 1; + continue :loop tags[inst]; + }, + .error_name_table_ref => { + wasm.error_name_table_ref_count += 1; + try code.ensureUnusedCapacity(gpa, 11); + const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; + code.appendAssumeCapacity(@intFromEnum(opcode)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.errorNameTableSymbolIndex() }, + .tag = if (is_wasm32) .memory_addr_leb else .memory_addr_leb64, + .addend = 0, + }); + code.appendNTimesAssumeCapacity(0, if (is_wasm32) 5 else 10); + + inst += 1; + continue :loop tags[inst]; + } else { + const addr: u32 = wasm.errorNameTableAddr(); + writeSleb128(code, addr); + + inst += 1; + continue :loop tags[inst]; + } + }, + .br_if, .br, .memory_grow, .memory_size => { + try code.ensureUnusedCapacity(gpa, 11); + code.appendAssumeCapacity(@intFromEnum(tags[inst])); + writeUleb128(code, datas[inst].label); + + inst += 1; + continue :loop tags[inst]; + }, + + .local_get, .local_set, .local_tee => { + try code.ensureUnusedCapacity(gpa, 11); + code.appendAssumeCapacity(@intFromEnum(tags[inst])); + writeUleb128(code, datas[inst].local); + + inst += 1; + continue :loop tags[inst]; + }, + + .br_table => { + const extra_index = datas[inst].payload; + const extra = mir.extraData(Mir.JumpTable, extra_index); + const labels = mir.extra[extra.end..][0..extra.data.length]; + try code.ensureUnusedCapacity(gpa, 11 + 10 * labels.len); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.br_table)); + // -1 because default label is not part of length/depth. + writeUleb128(code, extra.data.length - 1); + for (labels) |label| writeUleb128(code, label); + + inst += 1; + continue :loop tags[inst]; + }, + + .call_nav => { + try code.ensureUnusedCapacity(gpa, 6); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.navSymbolIndex(datas[inst].nav_index) }, + .tag = .function_index_leb, + .addend = 0, + }); + code.appendNTimesAssumeCapacity(0, 5); + } else { + appendOutputFunctionIndex(code, .fromIpNav(wasm, datas[inst].nav_index)); + } + + inst += 1; + continue :loop tags[inst]; + }, + + .call_indirect => { + try code.ensureUnusedCapacity(gpa, 11); + const fn_info = comp.zcu.?.typeToFunc(.fromInterned(datas[inst].ip_index)).?; + const func_ty_index = wasm.getExistingFunctionType( + fn_info.cc, + fn_info.param_types.get(&comp.zcu.?.intern_pool), + .fromInterned(fn_info.return_type), + target, + ).?; + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call_indirect)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .type_index = func_ty_index }, + .tag = .type_index_leb, + .addend = 0, + }); + code.appendNTimesAssumeCapacity(0, 5); + } else { + const index: Wasm.Flush.FuncTypeIndex = .fromTypeIndex(func_ty_index, &wasm.flush_buffer); + writeUleb128(code, @intFromEnum(index)); + } + writeUleb128(code, @as(u32, 0)); // table index + + inst += 1; + continue :loop tags[inst]; + }, + + .call_tag_name => { + try code.ensureUnusedCapacity(gpa, 6); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.tagNameSymbolIndex(datas[inst].ip_index) }, + .tag = .function_index_leb, + .addend = 0, + }); + code.appendNTimesAssumeCapacity(0, 5); + } else { + appendOutputFunctionIndex(code, .fromTagNameType(wasm, datas[inst].ip_index)); + } + + inst += 1; + continue :loop tags[inst]; + }, + + .call_intrinsic => { + // Although this currently uses `wasm.internString`, note that it + // *could* be changed to directly index into a preloaded strings + // table initialized based on the `Mir.Intrinsic` enum. + const symbol_name = try wasm.internString(@tagName(datas[inst].intrinsic)); + + try code.ensureUnusedCapacity(gpa, 6); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.call)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.symbolNameIndex(symbol_name) }, + .tag = .function_index_leb, + .addend = 0, + }); + code.appendNTimesAssumeCapacity(0, 5); + } else { + appendOutputFunctionIndex(code, .fromSymbolName(wasm, symbol_name)); + } + + inst += 1; + continue :loop tags[inst]; + }, + + .global_set_sp => { + try code.ensureUnusedCapacity(gpa, 6); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.global_set)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.stackPointerSymbolIndex() }, + .tag = .global_index_leb, + .addend = 0, + }); + code.appendNTimesAssumeCapacity(0, 5); + } else { + const sp_global: Wasm.GlobalIndex = .stack_pointer; + writeUleb128(code, @intFromEnum(sp_global)); + } + + inst += 1; + continue :loop tags[inst]; + }, + + .f32_const => { + try code.ensureUnusedCapacity(gpa, 5); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.f32_const)); + std.mem.writeInt(u32, code.addManyAsArrayAssumeCapacity(4), @bitCast(datas[inst].float32), .little); + + inst += 1; + continue :loop tags[inst]; + }, + + .f64_const => { + try code.ensureUnusedCapacity(gpa, 9); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.f64_const)); + const float64 = mir.extraData(Mir.Float64, datas[inst].payload).data; + std.mem.writeInt(u64, code.addManyAsArrayAssumeCapacity(8), float64.toInt(), .little); + + inst += 1; + continue :loop tags[inst]; + }, + .i32_const => { + try code.ensureUnusedCapacity(gpa, 6); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i32_const)); + writeSleb128(code, datas[inst].imm32); + + inst += 1; + continue :loop tags[inst]; + }, + .i64_const => { + try code.ensureUnusedCapacity(gpa, 11); + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.i64_const)); + const int64: i64 = @bitCast(mir.extraData(Mir.Imm64, datas[inst].payload).data.toInt()); + writeSleb128(code, int64); + + inst += 1; + continue :loop tags[inst]; + }, + + .i32_load, + .i64_load, + .f32_load, + .f64_load, + .i32_load8_s, + .i32_load8_u, + .i32_load16_s, + .i32_load16_u, + .i64_load8_s, + .i64_load8_u, + .i64_load16_s, + .i64_load16_u, + .i64_load32_s, + .i64_load32_u, + .i32_store, + .i64_store, + .f32_store, + .f64_store, + .i32_store8, + .i32_store16, + .i64_store8, + .i64_store16, + .i64_store32, + => { + try code.ensureUnusedCapacity(gpa, 1 + 20); + code.appendAssumeCapacity(@intFromEnum(tags[inst])); + encodeMemArg(code, mir.extraData(Mir.MemArg, datas[inst].payload).data); + inst += 1; + continue :loop tags[inst]; + }, + + .end, + .@"return", + .@"unreachable", + .select, + .i32_eqz, + .i32_eq, + .i32_ne, + .i32_lt_s, + .i32_lt_u, + .i32_gt_s, + .i32_gt_u, + .i32_le_s, + .i32_le_u, + .i32_ge_s, + .i32_ge_u, + .i64_eqz, + .i64_eq, + .i64_ne, + .i64_lt_s, + .i64_lt_u, + .i64_gt_s, + .i64_gt_u, + .i64_le_s, + .i64_le_u, + .i64_ge_s, + .i64_ge_u, + .f32_eq, + .f32_ne, + .f32_lt, + .f32_gt, + .f32_le, + .f32_ge, + .f64_eq, + .f64_ne, + .f64_lt, + .f64_gt, + .f64_le, + .f64_ge, + .i32_add, + .i32_sub, + .i32_mul, + .i32_div_s, + .i32_div_u, + .i32_and, + .i32_or, + .i32_xor, + .i32_shl, + .i32_shr_s, + .i32_shr_u, + .i64_add, + .i64_sub, + .i64_mul, + .i64_div_s, + .i64_div_u, + .i64_and, + .i64_or, + .i64_xor, + .i64_shl, + .i64_shr_s, + .i64_shr_u, + .f32_abs, + .f32_neg, + .f32_ceil, + .f32_floor, + .f32_trunc, + .f32_nearest, + .f32_sqrt, + .f32_add, + .f32_sub, + .f32_mul, + .f32_div, + .f32_min, + .f32_max, + .f32_copysign, + .f64_abs, + .f64_neg, + .f64_ceil, + .f64_floor, + .f64_trunc, + .f64_nearest, + .f64_sqrt, + .f64_add, + .f64_sub, + .f64_mul, + .f64_div, + .f64_min, + .f64_max, + .f64_copysign, + .i32_wrap_i64, + .i64_extend_i32_s, + .i64_extend_i32_u, + .i32_extend8_s, + .i32_extend16_s, + .i64_extend8_s, + .i64_extend16_s, + .i64_extend32_s, + .f32_demote_f64, + .f64_promote_f32, + .i32_reinterpret_f32, + .i64_reinterpret_f64, + .f32_reinterpret_i32, + .f64_reinterpret_i64, + .i32_trunc_f32_s, + .i32_trunc_f32_u, + .i32_trunc_f64_s, + .i32_trunc_f64_u, + .i64_trunc_f32_s, + .i64_trunc_f32_u, + .i64_trunc_f64_s, + .i64_trunc_f64_u, + .f32_convert_i32_s, + .f32_convert_i32_u, + .f32_convert_i64_s, + .f32_convert_i64_u, + .f64_convert_i32_s, + .f64_convert_i32_u, + .f64_convert_i64_s, + .f64_convert_i64_u, + .i32_rem_s, + .i32_rem_u, + .i64_rem_s, + .i64_rem_u, + .i32_popcnt, + .i64_popcnt, + .i32_clz, + .i32_ctz, + .i64_clz, + .i64_ctz, + => { + try code.append(gpa, @intFromEnum(tags[inst])); + inst += 1; + continue :loop tags[inst]; + }, + + .misc_prefix => { + try code.ensureUnusedCapacity(gpa, 6 + 6); + const extra_index = datas[inst].payload; + const opcode = mir.extra[extra_index]; + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.misc_prefix)); + writeUleb128(code, opcode); + switch (@as(std.wasm.MiscOpcode, @enumFromInt(opcode))) { + // bulk-memory opcodes + .data_drop => { + const segment = mir.extra[extra_index + 1]; + writeUleb128(code, segment); + + inst += 1; + continue :loop tags[inst]; + }, + .memory_init => { + const segment = mir.extra[extra_index + 1]; + writeUleb128(code, segment); + writeUleb128(code, @as(u32, 0)); // memory index + + inst += 1; + continue :loop tags[inst]; + }, + .memory_fill => { + writeUleb128(code, @as(u32, 0)); // memory index + + inst += 1; + continue :loop tags[inst]; + }, + .memory_copy => { + writeUleb128(code, @as(u32, 0)); // dst memory index + writeUleb128(code, @as(u32, 0)); // src memory index + + inst += 1; + continue :loop tags[inst]; + }, + + // nontrapping-float-to-int-conversion opcodes + .i32_trunc_sat_f32_s, + .i32_trunc_sat_f32_u, + .i32_trunc_sat_f64_s, + .i32_trunc_sat_f64_u, + .i64_trunc_sat_f32_s, + .i64_trunc_sat_f32_u, + .i64_trunc_sat_f64_s, + .i64_trunc_sat_f64_u, + => { + inst += 1; + continue :loop tags[inst]; + }, + + .table_init => @panic("TODO"), + .elem_drop => @panic("TODO"), + .table_copy => @panic("TODO"), + .table_grow => @panic("TODO"), + .table_size => @panic("TODO"), + .table_fill => @panic("TODO"), + + _ => unreachable, + } + comptime unreachable; + }, + .simd_prefix => { + try code.ensureUnusedCapacity(gpa, 6 + 20); + const extra_index = datas[inst].payload; + const opcode = mir.extra[extra_index]; + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.simd_prefix)); + writeUleb128(code, opcode); + switch (@as(std.wasm.SimdOpcode, @enumFromInt(opcode))) { + .v128_store, + .v128_load, + .v128_load8_splat, + .v128_load16_splat, + .v128_load32_splat, + .v128_load64_splat, + => { + encodeMemArg(code, mir.extraData(Mir.MemArg, extra_index + 1).data); + inst += 1; + continue :loop tags[inst]; + }, + .v128_const, .i8x16_shuffle => { + code.appendSliceAssumeCapacity(std.mem.asBytes(mir.extra[extra_index + 1 ..][0..4])); + inst += 1; + continue :loop tags[inst]; + }, + .i8x16_extract_lane_s, + .i8x16_extract_lane_u, + .i8x16_replace_lane, + .i16x8_extract_lane_s, + .i16x8_extract_lane_u, + .i16x8_replace_lane, + .i32x4_extract_lane, + .i32x4_replace_lane, + .i64x2_extract_lane, + .i64x2_replace_lane, + .f32x4_extract_lane, + .f32x4_replace_lane, + .f64x2_extract_lane, + .f64x2_replace_lane, + => { + code.appendAssumeCapacity(@intCast(mir.extra[extra_index + 1])); + inst += 1; + continue :loop tags[inst]; + }, + .i8x16_splat, + .i16x8_splat, + .i32x4_splat, + .i64x2_splat, + .f32x4_splat, + .f64x2_splat, + => { + inst += 1; + continue :loop tags[inst]; + }, + + .v128_load8x8_s => @panic("TODO"), + .v128_load8x8_u => @panic("TODO"), + .v128_load16x4_s => @panic("TODO"), + .v128_load16x4_u => @panic("TODO"), + .v128_load32x2_s => @panic("TODO"), + .v128_load32x2_u => @panic("TODO"), + .i8x16_swizzle => @panic("TODO"), + .i8x16_eq => @panic("TODO"), + .i16x8_eq => @panic("TODO"), + .i32x4_eq => @panic("TODO"), + .i8x16_ne => @panic("TODO"), + .i16x8_ne => @panic("TODO"), + .i32x4_ne => @panic("TODO"), + .i8x16_lt_s => @panic("TODO"), + .i16x8_lt_s => @panic("TODO"), + .i32x4_lt_s => @panic("TODO"), + .i8x16_lt_u => @panic("TODO"), + .i16x8_lt_u => @panic("TODO"), + .i32x4_lt_u => @panic("TODO"), + .i8x16_gt_s => @panic("TODO"), + .i16x8_gt_s => @panic("TODO"), + .i32x4_gt_s => @panic("TODO"), + .i8x16_gt_u => @panic("TODO"), + .i16x8_gt_u => @panic("TODO"), + .i32x4_gt_u => @panic("TODO"), + .i8x16_le_s => @panic("TODO"), + .i16x8_le_s => @panic("TODO"), + .i32x4_le_s => @panic("TODO"), + .i8x16_le_u => @panic("TODO"), + .i16x8_le_u => @panic("TODO"), + .i32x4_le_u => @panic("TODO"), + .i8x16_ge_s => @panic("TODO"), + .i16x8_ge_s => @panic("TODO"), + .i32x4_ge_s => @panic("TODO"), + .i8x16_ge_u => @panic("TODO"), + .i16x8_ge_u => @panic("TODO"), + .i32x4_ge_u => @panic("TODO"), + .f32x4_eq => @panic("TODO"), + .f64x2_eq => @panic("TODO"), + .f32x4_ne => @panic("TODO"), + .f64x2_ne => @panic("TODO"), + .f32x4_lt => @panic("TODO"), + .f64x2_lt => @panic("TODO"), + .f32x4_gt => @panic("TODO"), + .f64x2_gt => @panic("TODO"), + .f32x4_le => @panic("TODO"), + .f64x2_le => @panic("TODO"), + .f32x4_ge => @panic("TODO"), + .f64x2_ge => @panic("TODO"), + .v128_not => @panic("TODO"), + .v128_and => @panic("TODO"), + .v128_andnot => @panic("TODO"), + .v128_or => @panic("TODO"), + .v128_xor => @panic("TODO"), + .v128_bitselect => @panic("TODO"), + .v128_any_true => @panic("TODO"), + .v128_load8_lane => @panic("TODO"), + .v128_load16_lane => @panic("TODO"), + .v128_load32_lane => @panic("TODO"), + .v128_load64_lane => @panic("TODO"), + .v128_store8_lane => @panic("TODO"), + .v128_store16_lane => @panic("TODO"), + .v128_store32_lane => @panic("TODO"), + .v128_store64_lane => @panic("TODO"), + .v128_load32_zero => @panic("TODO"), + .v128_load64_zero => @panic("TODO"), + .f32x4_demote_f64x2_zero => @panic("TODO"), + .f64x2_promote_low_f32x4 => @panic("TODO"), + .i8x16_abs => @panic("TODO"), + .i16x8_abs => @panic("TODO"), + .i32x4_abs => @panic("TODO"), + .i64x2_abs => @panic("TODO"), + .i8x16_neg => @panic("TODO"), + .i16x8_neg => @panic("TODO"), + .i32x4_neg => @panic("TODO"), + .i64x2_neg => @panic("TODO"), + .i8x16_popcnt => @panic("TODO"), + .i16x8_q15mulr_sat_s => @panic("TODO"), + .i8x16_all_true => @panic("TODO"), + .i16x8_all_true => @panic("TODO"), + .i32x4_all_true => @panic("TODO"), + .i64x2_all_true => @panic("TODO"), + .i8x16_bitmask => @panic("TODO"), + .i16x8_bitmask => @panic("TODO"), + .i32x4_bitmask => @panic("TODO"), + .i64x2_bitmask => @panic("TODO"), + .i8x16_narrow_i16x8_s => @panic("TODO"), + .i16x8_narrow_i32x4_s => @panic("TODO"), + .i8x16_narrow_i16x8_u => @panic("TODO"), + .i16x8_narrow_i32x4_u => @panic("TODO"), + .f32x4_ceil => @panic("TODO"), + .i16x8_extend_low_i8x16_s => @panic("TODO"), + .i32x4_extend_low_i16x8_s => @panic("TODO"), + .i64x2_extend_low_i32x4_s => @panic("TODO"), + .f32x4_floor => @panic("TODO"), + .i16x8_extend_high_i8x16_s => @panic("TODO"), + .i32x4_extend_high_i16x8_s => @panic("TODO"), + .i64x2_extend_high_i32x4_s => @panic("TODO"), + .f32x4_trunc => @panic("TODO"), + .i16x8_extend_low_i8x16_u => @panic("TODO"), + .i32x4_extend_low_i16x8_u => @panic("TODO"), + .i64x2_extend_low_i32x4_u => @panic("TODO"), + .f32x4_nearest => @panic("TODO"), + .i16x8_extend_high_i8x16_u => @panic("TODO"), + .i32x4_extend_high_i16x8_u => @panic("TODO"), + .i64x2_extend_high_i32x4_u => @panic("TODO"), + .i8x16_shl => @panic("TODO"), + .i16x8_shl => @panic("TODO"), + .i32x4_shl => @panic("TODO"), + .i64x2_shl => @panic("TODO"), + .i8x16_shr_s => @panic("TODO"), + .i16x8_shr_s => @panic("TODO"), + .i32x4_shr_s => @panic("TODO"), + .i64x2_shr_s => @panic("TODO"), + .i8x16_shr_u => @panic("TODO"), + .i16x8_shr_u => @panic("TODO"), + .i32x4_shr_u => @panic("TODO"), + .i64x2_shr_u => @panic("TODO"), + .i8x16_add => @panic("TODO"), + .i16x8_add => @panic("TODO"), + .i32x4_add => @panic("TODO"), + .i64x2_add => @panic("TODO"), + .i8x16_add_sat_s => @panic("TODO"), + .i16x8_add_sat_s => @panic("TODO"), + .i8x16_add_sat_u => @panic("TODO"), + .i16x8_add_sat_u => @panic("TODO"), + .i8x16_sub => @panic("TODO"), + .i16x8_sub => @panic("TODO"), + .i32x4_sub => @panic("TODO"), + .i64x2_sub => @panic("TODO"), + .i8x16_sub_sat_s => @panic("TODO"), + .i16x8_sub_sat_s => @panic("TODO"), + .i8x16_sub_sat_u => @panic("TODO"), + .i16x8_sub_sat_u => @panic("TODO"), + .f64x2_ceil => @panic("TODO"), + .f64x2_nearest => @panic("TODO"), + .f64x2_floor => @panic("TODO"), + .i16x8_mul => @panic("TODO"), + .i32x4_mul => @panic("TODO"), + .i64x2_mul => @panic("TODO"), + .i8x16_min_s => @panic("TODO"), + .i16x8_min_s => @panic("TODO"), + .i32x4_min_s => @panic("TODO"), + .i64x2_eq => @panic("TODO"), + .i8x16_min_u => @panic("TODO"), + .i16x8_min_u => @panic("TODO"), + .i32x4_min_u => @panic("TODO"), + .i64x2_ne => @panic("TODO"), + .i8x16_max_s => @panic("TODO"), + .i16x8_max_s => @panic("TODO"), + .i32x4_max_s => @panic("TODO"), + .i64x2_lt_s => @panic("TODO"), + .i8x16_max_u => @panic("TODO"), + .i16x8_max_u => @panic("TODO"), + .i32x4_max_u => @panic("TODO"), + .i64x2_gt_s => @panic("TODO"), + .f64x2_trunc => @panic("TODO"), + .i32x4_dot_i16x8_s => @panic("TODO"), + .i64x2_le_s => @panic("TODO"), + .i8x16_avgr_u => @panic("TODO"), + .i16x8_avgr_u => @panic("TODO"), + .i64x2_ge_s => @panic("TODO"), + .i16x8_extadd_pairwise_i8x16_s => @panic("TODO"), + .i16x8_extmul_low_i8x16_s => @panic("TODO"), + .i32x4_extmul_low_i16x8_s => @panic("TODO"), + .i64x2_extmul_low_i32x4_s => @panic("TODO"), + .i16x8_extadd_pairwise_i8x16_u => @panic("TODO"), + .i16x8_extmul_high_i8x16_s => @panic("TODO"), + .i32x4_extmul_high_i16x8_s => @panic("TODO"), + .i64x2_extmul_high_i32x4_s => @panic("TODO"), + .i32x4_extadd_pairwise_i16x8_s => @panic("TODO"), + .i16x8_extmul_low_i8x16_u => @panic("TODO"), + .i32x4_extmul_low_i16x8_u => @panic("TODO"), + .i64x2_extmul_low_i32x4_u => @panic("TODO"), + .i32x4_extadd_pairwise_i16x8_u => @panic("TODO"), + .i16x8_extmul_high_i8x16_u => @panic("TODO"), + .i32x4_extmul_high_i16x8_u => @panic("TODO"), + .i64x2_extmul_high_i32x4_u => @panic("TODO"), + .f32x4_abs => @panic("TODO"), + .f64x2_abs => @panic("TODO"), + .f32x4_neg => @panic("TODO"), + .f64x2_neg => @panic("TODO"), + .f32x4_sqrt => @panic("TODO"), + .f64x2_sqrt => @panic("TODO"), + .f32x4_add => @panic("TODO"), + .f64x2_add => @panic("TODO"), + .f32x4_sub => @panic("TODO"), + .f64x2_sub => @panic("TODO"), + .f32x4_mul => @panic("TODO"), + .f64x2_mul => @panic("TODO"), + .f32x4_div => @panic("TODO"), + .f64x2_div => @panic("TODO"), + .f32x4_min => @panic("TODO"), + .f64x2_min => @panic("TODO"), + .f32x4_max => @panic("TODO"), + .f64x2_max => @panic("TODO"), + .f32x4_pmin => @panic("TODO"), + .f64x2_pmin => @panic("TODO"), + .f32x4_pmax => @panic("TODO"), + .f64x2_pmax => @panic("TODO"), + .i32x4_trunc_sat_f32x4_s => @panic("TODO"), + .i32x4_trunc_sat_f32x4_u => @panic("TODO"), + .f32x4_convert_i32x4_s => @panic("TODO"), + .f32x4_convert_i32x4_u => @panic("TODO"), + .i32x4_trunc_sat_f64x2_s_zero => @panic("TODO"), + .i32x4_trunc_sat_f64x2_u_zero => @panic("TODO"), + .f64x2_convert_low_i32x4_s => @panic("TODO"), + .f64x2_convert_low_i32x4_u => @panic("TODO"), + .i8x16_relaxed_swizzle => @panic("TODO"), + .i32x4_relaxed_trunc_f32x4_s => @panic("TODO"), + .i32x4_relaxed_trunc_f32x4_u => @panic("TODO"), + .i32x4_relaxed_trunc_f64x2_s_zero => @panic("TODO"), + .i32x4_relaxed_trunc_f64x2_u_zero => @panic("TODO"), + .f32x4_relaxed_madd => @panic("TODO"), + .f32x4_relaxed_nmadd => @panic("TODO"), + .f64x2_relaxed_madd => @panic("TODO"), + .f64x2_relaxed_nmadd => @panic("TODO"), + .i8x16_relaxed_laneselect => @panic("TODO"), + .i16x8_relaxed_laneselect => @panic("TODO"), + .i32x4_relaxed_laneselect => @panic("TODO"), + .i64x2_relaxed_laneselect => @panic("TODO"), + .f32x4_relaxed_min => @panic("TODO"), + .f32x4_relaxed_max => @panic("TODO"), + .f64x2_relaxed_min => @panic("TODO"), + .f64x2_relaxed_max => @panic("TODO"), + .i16x8_relaxed_q15mulr_s => @panic("TODO"), + .i16x8_relaxed_dot_i8x16_i7x16_s => @panic("TODO"), + .i32x4_relaxed_dot_i8x16_i7x16_add_s => @panic("TODO"), + .f32x4_relaxed_dot_bf16x8_add_f32x4 => @panic("TODO"), + } + comptime unreachable; + }, + .atomics_prefix => { + try code.ensureUnusedCapacity(gpa, 6 + 20); + + const extra_index = datas[inst].payload; + const opcode = mir.extra[extra_index]; + code.appendAssumeCapacity(@intFromEnum(std.wasm.Opcode.atomics_prefix)); + writeUleb128(code, opcode); + switch (@as(std.wasm.AtomicsOpcode, @enumFromInt(opcode))) { + .i32_atomic_load, + .i64_atomic_load, + .i32_atomic_load8_u, + .i32_atomic_load16_u, + .i64_atomic_load8_u, + .i64_atomic_load16_u, + .i64_atomic_load32_u, + .i32_atomic_store, + .i64_atomic_store, + .i32_atomic_store8, + .i32_atomic_store16, + .i64_atomic_store8, + .i64_atomic_store16, + .i64_atomic_store32, + .i32_atomic_rmw_add, + .i64_atomic_rmw_add, + .i32_atomic_rmw8_add_u, + .i32_atomic_rmw16_add_u, + .i64_atomic_rmw8_add_u, + .i64_atomic_rmw16_add_u, + .i64_atomic_rmw32_add_u, + .i32_atomic_rmw_sub, + .i64_atomic_rmw_sub, + .i32_atomic_rmw8_sub_u, + .i32_atomic_rmw16_sub_u, + .i64_atomic_rmw8_sub_u, + .i64_atomic_rmw16_sub_u, + .i64_atomic_rmw32_sub_u, + .i32_atomic_rmw_and, + .i64_atomic_rmw_and, + .i32_atomic_rmw8_and_u, + .i32_atomic_rmw16_and_u, + .i64_atomic_rmw8_and_u, + .i64_atomic_rmw16_and_u, + .i64_atomic_rmw32_and_u, + .i32_atomic_rmw_or, + .i64_atomic_rmw_or, + .i32_atomic_rmw8_or_u, + .i32_atomic_rmw16_or_u, + .i64_atomic_rmw8_or_u, + .i64_atomic_rmw16_or_u, + .i64_atomic_rmw32_or_u, + .i32_atomic_rmw_xor, + .i64_atomic_rmw_xor, + .i32_atomic_rmw8_xor_u, + .i32_atomic_rmw16_xor_u, + .i64_atomic_rmw8_xor_u, + .i64_atomic_rmw16_xor_u, + .i64_atomic_rmw32_xor_u, + .i32_atomic_rmw_xchg, + .i64_atomic_rmw_xchg, + .i32_atomic_rmw8_xchg_u, + .i32_atomic_rmw16_xchg_u, + .i64_atomic_rmw8_xchg_u, + .i64_atomic_rmw16_xchg_u, + .i64_atomic_rmw32_xchg_u, + + .i32_atomic_rmw_cmpxchg, + .i64_atomic_rmw_cmpxchg, + .i32_atomic_rmw8_cmpxchg_u, + .i32_atomic_rmw16_cmpxchg_u, + .i64_atomic_rmw8_cmpxchg_u, + .i64_atomic_rmw16_cmpxchg_u, + .i64_atomic_rmw32_cmpxchg_u, + => { + const mem_arg = mir.extraData(Mir.MemArg, extra_index + 1).data; + encodeMemArg(code, mem_arg); + inst += 1; + continue :loop tags[inst]; + }, + .atomic_fence => { + // Hard-codes memory index 0 since multi-memory proposal is + // not yet accepted nor implemented. + const memory_index: u32 = 0; + writeUleb128(code, memory_index); + inst += 1; + continue :loop tags[inst]; + }, + .memory_atomic_notify => @panic("TODO"), + .memory_atomic_wait32 => @panic("TODO"), + .memory_atomic_wait64 => @panic("TODO"), + } + comptime unreachable; + }, + } + comptime unreachable; +} + +/// Asserts 20 unused capacity. +fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void { + assert(code.unusedCapacitySlice().len >= 20); + // Wasm encodes alignment as power of 2, rather than natural alignment. + const encoded_alignment = @ctz(mem_arg.alignment); + writeUleb128(code, encoded_alignment); + writeUleb128(code, mem_arg.offset); +} + +fn uavRefObj(wasm: *Wasm, code: *ArrayList(u8), value: InternPool.Index, offset: i32, is_wasm32: bool) !void { + const comp = wasm.base.comp; + const gpa = comp.gpa; + const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; + + try code.ensureUnusedCapacity(gpa, 11); + code.appendAssumeCapacity(@intFromEnum(opcode)); + + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.uavSymbolIndex(value) }, + .tag = if (is_wasm32) .memory_addr_leb else .memory_addr_leb64, + .addend = offset, + }); + code.appendNTimesAssumeCapacity(0, if (is_wasm32) 5 else 10); +} + +fn uavRefExe(wasm: *Wasm, code: *ArrayList(u8), value: InternPool.Index, offset: i32, is_wasm32: bool) !void { + const comp = wasm.base.comp; + const gpa = comp.gpa; + const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; + + try code.ensureUnusedCapacity(gpa, 11); + code.appendAssumeCapacity(@intFromEnum(opcode)); + + const addr = wasm.uavAddr(value); + writeUleb128(code, @as(u32, @intCast(@as(i64, addr) + offset))); +} + +fn navRefOff(wasm: *Wasm, code: *ArrayList(u8), data: Mir.NavRefOff, is_wasm32: bool) !void { + const comp = wasm.base.comp; + const zcu = comp.zcu.?; + const ip = &zcu.intern_pool; + const gpa = comp.gpa; + const is_obj = comp.config.output_mode == .Obj; + const nav_ty = ip.getNav(data.nav_index).typeOf(ip); + assert(!ip.isFunctionType(nav_ty)); + + try code.ensureUnusedCapacity(gpa, 11); + + const opcode: std.wasm.Opcode = if (is_wasm32) .i32_const else .i64_const; + code.appendAssumeCapacity(@intFromEnum(opcode)); + if (is_obj) { + try wasm.out_relocs.append(gpa, .{ + .offset = @intCast(code.items.len), + .pointee = .{ .symbol_index = try wasm.navSymbolIndex(data.nav_index) }, + .tag = if (is_wasm32) .memory_addr_leb else .memory_addr_leb64, + .addend = data.offset, + }); + code.appendNTimesAssumeCapacity(0, if (is_wasm32) 5 else 10); + } else { + const addr = wasm.navAddr(data.nav_index); + writeUleb128(code, @as(u32, @intCast(@as(i64, addr) + data.offset))); + } +} + +fn appendOutputFunctionIndex(code: *ArrayList(u8), i: Wasm.OutputFunctionIndex) void { + writeUleb128(code, @intFromEnum(i)); +} + +fn writeUleb128(code: *ArrayList(u8), arg: anytype) void { + var w: std.Io.Writer = .fixed(code.unusedCapacitySlice()); + w.writeUleb128(arg) catch unreachable; + code.items.len += w.end; +} + +fn writeSleb128(code: *ArrayList(u8), arg: anytype) void { + var w: std.Io.Writer = .fixed(code.unusedCapacitySlice()); + w.writeSleb128(arg) catch unreachable; + code.items.len += w.end; +} diff --git a/src/codegen/wasm/Mir.zig b/src/codegen/wasm/Mir.zig new file mode 100644 index 0000000000000000000000000000000000000000..af6e0eb40b590cfe486c246410b455af582f73d3 --- /dev/null +++ b/src/codegen/wasm/Mir.zig @@ -0,0 +1,1003 @@ +//! Machine Intermediate Representation. +//! This representation is produced by wasm Codegen. +//! Each of these instructions have a 1:1 mapping to a wasm opcode, +//! but may contain metadata for a specific opcode such as an immediate. +//! MIR can be lowered to both textual code (wat) and binary format (wasm). +//! The main benefits of MIR is optimization passes, pre-allocated locals, +//! and known jump labels for blocks. + +const Mir = @This(); +const InternPool = @import("../../InternPool.zig"); +const Wasm = @import("../../link/Wasm.zig"); +const Emit = @import("Emit.zig"); +const Alignment = InternPool.Alignment; + +const builtin = @import("builtin"); +const std = @import("std"); +const assert = std.debug.assert; +const leb = std.leb; + +instructions: std.MultiArrayList(Inst).Slice, +/// A slice of indexes where the meaning of the data is determined by the +/// `Inst.Tag` value. +extra: []const u32, +locals: []const std.wasm.Valtype, +prologue: Prologue, + +/// Not directly used by `Emit`, but the linker needs this to merge it with a global set. +/// Value is the explicit alignment if greater than natural alignment, `.none` otherwise. +uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment), +/// Not directly used by `Emit`, but the linker needs this to merge it with a global set. +indirect_function_set: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, void), +/// Not directly used by `Emit`, but the linker needs this to ensure these types are interned. +func_tys: std.AutoArrayHashMapUnmanaged(InternPool.Index, void), +/// Not directly used by `Emit`, but the linker needs this to add it to its own refcount. +error_name_table_ref_count: u32, + +pub const Prologue = extern struct { + flags: Flags, + sp_local: u32, + stack_size: u32, + bottom_stack_local: u32, + + pub const Flags = packed struct(u32) { + stack_alignment: Alignment, + padding: u26 = 0, + }; + + pub const none: Prologue = .{ + .sp_local = 0, + .flags = .{ .stack_alignment = .none }, + .stack_size = 0, + .bottom_stack_local = 0, + }; + + pub fn isNone(p: *const Prologue) bool { + return p.flags.stack_alignment != .none; + } +}; + +pub const Inst = struct { + /// The opcode that represents this instruction + tag: Tag, + /// Data is determined by the set `tag`. + /// For example, `data` will be an i32 for when `tag` is 'i32_const'. + data: Data, + + /// The position of a given MIR isntruction with the instruction list. + pub const Index = u32; + + /// Some tags match wasm opcode values to facilitate trivial lowering. + pub const Tag = enum(u8) { + /// Uses `tag`. + @"unreachable" = 0x00, + /// Emits epilogue begin debug information. Marks the end of the function. + /// + /// Uses `tag` (no additional data). + dbg_epilogue_begin, + /// Creates a new block that can be jump from. + /// + /// Type of the block is given in data `block_type` + block = 0x02, + /// Creates a new loop. + /// + /// Type of the loop is given in data `block_type` + loop = 0x03, + /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the + /// memory address of an unnamed constant. When emitting an object + /// file, this adds a relocation. + /// + /// This may not refer to a function. + /// + /// Uses `ip_index`. + uav_ref, + /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the + /// memory address of an unnamed constant, offset by an integer value. + /// When emitting an object file, this adds a relocation. + /// + /// This may not refer to a function. + /// + /// Uses `payload` pointing to a `UavRefOff`. + uav_ref_off, + /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the + /// memory address of a named constant. + /// + /// May not refer to a function. + /// + /// Uses `nav_index`. + nav_ref, + /// Lowers to an i32_const (wasm32) or i64_const (wasm64) which is the + /// memory address of named constant, offset by an integer value. + /// When emitting an object file, this adds a relocation. + /// + /// May not refer to a function. + /// + /// Uses `payload` pointing to a `NavRefOff`. + nav_ref_off, + /// Lowers to an i32_const which is the index of the function in the + /// table section. + /// + /// Uses `nav_index`. + func_ref, + /// Inserts debug information about the current line and column + /// of the source code + /// + /// Uses `payload` of which the payload type is `DbgLineColumn` + dbg_line, + /// Lowers to an i32_const containing the number of unique Zig error + /// names. + /// Uses `tag`. + errors_len, + /// Represents the end of a function body or an initialization expression + /// + /// Uses `tag` (no additional data). + end = 0x0B, + /// Breaks from the current block to a label + /// + /// Uses `label` where index represents the label to jump to + br = 0x0C, + /// Breaks from the current block if the stack value is non-zero + /// + /// Uses `label` where index represents the label to jump to + br_if = 0x0D, + /// Jump table that takes the stack value as an index where each value + /// represents the label to jump to. + /// + /// Data is extra of which the Payload's type is `JumpTable` + br_table, + /// Returns from the function + /// + /// Uses `tag`. + @"return" = 0x0F, + /// Lowers to an i32_const (wasm32) or i64_const (wasm64) containing + /// the base address of the table of error code names, with each + /// element being a null-terminated slice. + /// + /// Uses `tag`. + error_name_table_ref, + /// Calls a function using `nav_index`. + call_nav, + /// Calls a function pointer by its function signature + /// and index into the function table. + /// + /// Uses `ip_index`; the `InternPool.Index` is the function type. + call_indirect, + /// Calls a function by its index. + /// + /// The function is the auto-generated tag name function for the type + /// provided in `ip_index`. + call_tag_name, + /// Lowers to a `call` instruction, using `intrinsic`. + call_intrinsic, + /// Pops three values from the stack and pushes + /// the first or second value dependent on the third value. + /// Uses `tag` + select = 0x1B, + /// Loads a local at given index onto the stack. + /// + /// Uses `label` + local_get = 0x20, + /// Pops a value from the stack into the local at given index. + /// Stack value must be of the same type as the local. + /// + /// Uses `label` + local_set = 0x21, + /// Sets a local at given index using the value at the top of the stack without popping the value. + /// Stack value must have the same type as the local. + /// + /// Uses `label` + local_tee = 0x22, + /// Pops a value from the stack and sets the stack pointer global. + /// The value must be the same type as the stack pointer global. + /// + /// Uses `tag` (no additional data). + global_set_sp, + /// Loads a 32-bit integer from memory (data section) onto the stack + /// Pops the value from the stack which represents the offset into memory. + /// + /// Uses `payload` of type `MemArg`. + i32_load = 0x28, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load = 0x29, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + f32_load = 0x2A, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + f64_load = 0x2B, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i32_load8_s = 0x2C, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i32_load8_u = 0x2D, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i32_load16_s = 0x2E, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i32_load16_u = 0x2F, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load8_s = 0x30, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load8_u = 0x31, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load16_s = 0x32, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load16_u = 0x33, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load32_s = 0x34, + /// Loads a value from memory onto the stack, based on the signedness + /// and bitsize of the type. + /// + /// Uses `payload` with type `MemArg` + i64_load32_u = 0x35, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `payload` of type `MemArg`. + i32_store = 0x36, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + i64_store = 0x37, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + f32_store = 0x38, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + f64_store = 0x39, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + i32_store8 = 0x3A, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + i32_store16 = 0x3B, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + i64_store8 = 0x3C, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + i64_store16 = 0x3D, + /// Pops 2 values from the stack, where the first value represents the value to write into memory + /// and the second value represents the offset into memory where the value must be written to. + /// This opcode is typed and expects the stack value's type to be equal to this opcode's type. + /// + /// Uses `Payload` with type `MemArg` + i64_store32 = 0x3E, + /// Returns the memory size in amount of pages. + /// + /// Uses `label` + memory_size = 0x3F, + /// Increases the memory by given number of pages. + /// + /// Uses `label` + memory_grow = 0x40, + /// Loads a 32-bit signed immediate value onto the stack + /// + /// Uses `imm32` + i32_const, + /// Loads a i64-bit signed immediate value onto the stack + /// + /// uses `payload` of type `Imm64` + i64_const, + /// Loads a 32-bit float value onto the stack. + /// + /// Uses `float32` + f32_const, + /// Loads a 64-bit float value onto the stack. + /// + /// Uses `payload` of type `Float64` + f64_const, + /// Uses `tag` + i32_eqz = 0x45, + /// Uses `tag` + i32_eq = 0x46, + /// Uses `tag` + i32_ne = 0x47, + /// Uses `tag` + i32_lt_s = 0x48, + /// Uses `tag` + i32_lt_u = 0x49, + /// Uses `tag` + i32_gt_s = 0x4A, + /// Uses `tag` + i32_gt_u = 0x4B, + /// Uses `tag` + i32_le_s = 0x4C, + /// Uses `tag` + i32_le_u = 0x4D, + /// Uses `tag` + i32_ge_s = 0x4E, + /// Uses `tag` + i32_ge_u = 0x4F, + /// Uses `tag` + i64_eqz = 0x50, + /// Uses `tag` + i64_eq = 0x51, + /// Uses `tag` + i64_ne = 0x52, + /// Uses `tag` + i64_lt_s = 0x53, + /// Uses `tag` + i64_lt_u = 0x54, + /// Uses `tag` + i64_gt_s = 0x55, + /// Uses `tag` + i64_gt_u = 0x56, + /// Uses `tag` + i64_le_s = 0x57, + /// Uses `tag` + i64_le_u = 0x58, + /// Uses `tag` + i64_ge_s = 0x59, + /// Uses `tag` + i64_ge_u = 0x5A, + /// Uses `tag` + f32_eq = 0x5B, + /// Uses `tag` + f32_ne = 0x5C, + /// Uses `tag` + f32_lt = 0x5D, + /// Uses `tag` + f32_gt = 0x5E, + /// Uses `tag` + f32_le = 0x5F, + /// Uses `tag` + f32_ge = 0x60, + /// Uses `tag` + f64_eq = 0x61, + /// Uses `tag` + f64_ne = 0x62, + /// Uses `tag` + f64_lt = 0x63, + /// Uses `tag` + f64_gt = 0x64, + /// Uses `tag` + f64_le = 0x65, + /// Uses `tag` + f64_ge = 0x66, + /// Uses `tag` + i32_clz = 0x67, + /// Uses `tag` + i32_ctz = 0x68, + /// Uses `tag` + i32_popcnt = 0x69, + /// Uses `tag` + i32_add = 0x6A, + /// Uses `tag` + i32_sub = 0x6B, + /// Uses `tag` + i32_mul = 0x6C, + /// Uses `tag` + i32_div_s = 0x6D, + /// Uses `tag` + i32_div_u = 0x6E, + /// Uses `tag` + i32_rem_s = 0x6F, + /// Uses `tag` + i32_rem_u = 0x70, + /// Uses `tag` + i32_and = 0x71, + /// Uses `tag` + i32_or = 0x72, + /// Uses `tag` + i32_xor = 0x73, + /// Uses `tag` + i32_shl = 0x74, + /// Uses `tag` + i32_shr_s = 0x75, + /// Uses `tag` + i32_shr_u = 0x76, + /// Uses `tag` + i64_clz = 0x79, + /// Uses `tag` + i64_ctz = 0x7A, + /// Uses `tag` + i64_popcnt = 0x7B, + /// Uses `tag` + i64_add = 0x7C, + /// Uses `tag` + i64_sub = 0x7D, + /// Uses `tag` + i64_mul = 0x7E, + /// Uses `tag` + i64_div_s = 0x7F, + /// Uses `tag` + i64_div_u = 0x80, + /// Uses `tag` + i64_rem_s = 0x81, + /// Uses `tag` + i64_rem_u = 0x82, + /// Uses `tag` + i64_and = 0x83, + /// Uses `tag` + i64_or = 0x84, + /// Uses `tag` + i64_xor = 0x85, + /// Uses `tag` + i64_shl = 0x86, + /// Uses `tag` + i64_shr_s = 0x87, + /// Uses `tag` + i64_shr_u = 0x88, + /// Uses `tag` + f32_abs = 0x8B, + /// Uses `tag` + f32_neg = 0x8C, + /// Uses `tag` + f32_ceil = 0x8D, + /// Uses `tag` + f32_floor = 0x8E, + /// Uses `tag` + f32_trunc = 0x8F, + /// Uses `tag` + f32_nearest = 0x90, + /// Uses `tag` + f32_sqrt = 0x91, + /// Uses `tag` + f32_add = 0x92, + /// Uses `tag` + f32_sub = 0x93, + /// Uses `tag` + f32_mul = 0x94, + /// Uses `tag` + f32_div = 0x95, + /// Uses `tag` + f32_min = 0x96, + /// Uses `tag` + f32_max = 0x97, + /// Uses `tag` + f32_copysign = 0x98, + /// Uses `tag` + f64_abs = 0x99, + /// Uses `tag` + f64_neg = 0x9A, + /// Uses `tag` + f64_ceil = 0x9B, + /// Uses `tag` + f64_floor = 0x9C, + /// Uses `tag` + f64_trunc = 0x9D, + /// Uses `tag` + f64_nearest = 0x9E, + /// Uses `tag` + f64_sqrt = 0x9F, + /// Uses `tag` + f64_add = 0xA0, + /// Uses `tag` + f64_sub = 0xA1, + /// Uses `tag` + f64_mul = 0xA2, + /// Uses `tag` + f64_div = 0xA3, + /// Uses `tag` + f64_min = 0xA4, + /// Uses `tag` + f64_max = 0xA5, + /// Uses `tag` + f64_copysign = 0xA6, + /// Uses `tag` + i32_wrap_i64 = 0xA7, + /// Uses `tag` + i32_trunc_f32_s = 0xA8, + /// Uses `tag` + i32_trunc_f32_u = 0xA9, + /// Uses `tag` + i32_trunc_f64_s = 0xAA, + /// Uses `tag` + i32_trunc_f64_u = 0xAB, + /// Uses `tag` + i64_extend_i32_s = 0xAC, + /// Uses `tag` + i64_extend_i32_u = 0xAD, + /// Uses `tag` + i64_trunc_f32_s = 0xAE, + /// Uses `tag` + i64_trunc_f32_u = 0xAF, + /// Uses `tag` + i64_trunc_f64_s = 0xB0, + /// Uses `tag` + i64_trunc_f64_u = 0xB1, + /// Uses `tag` + f32_convert_i32_s = 0xB2, + /// Uses `tag` + f32_convert_i32_u = 0xB3, + /// Uses `tag` + f32_convert_i64_s = 0xB4, + /// Uses `tag` + f32_convert_i64_u = 0xB5, + /// Uses `tag` + f32_demote_f64 = 0xB6, + /// Uses `tag` + f64_convert_i32_s = 0xB7, + /// Uses `tag` + f64_convert_i32_u = 0xB8, + /// Uses `tag` + f64_convert_i64_s = 0xB9, + /// Uses `tag` + f64_convert_i64_u = 0xBA, + /// Uses `tag` + f64_promote_f32 = 0xBB, + /// Uses `tag` + i32_reinterpret_f32 = 0xBC, + /// Uses `tag` + i64_reinterpret_f64 = 0xBD, + /// Uses `tag` + f32_reinterpret_i32 = 0xBE, + /// Uses `tag` + f64_reinterpret_i64 = 0xBF, + /// Uses `tag` + i32_extend8_s = 0xC0, + /// Uses `tag` + i32_extend16_s = 0xC1, + /// Uses `tag` + i64_extend8_s = 0xC2, + /// Uses `tag` + i64_extend16_s = 0xC3, + /// Uses `tag` + i64_extend32_s = 0xC4, + /// The instruction consists of a prefixed opcode. + /// The prefixed opcode can be found at payload's index. + /// + /// The `data` field depends on the extension instruction and + /// may contain additional data. + misc_prefix, + /// The instruction consists of a simd opcode. + /// The actual simd-opcode is found at payload's index. + /// + /// The `data` field depends on the simd instruction and + /// may contain additional data. + simd_prefix, + /// The instruction consists of an atomics opcode. + /// The actual atomics-opcode is found at payload's index. + /// + /// The `data` field depends on the atomics instruction and + /// may contain additional data. + atomics_prefix = 0xFE, + + /// From a given wasm opcode, returns a MIR tag. + pub fn fromOpcode(opcode: std.wasm.Opcode) Tag { + return @as(Tag, @enumFromInt(@intFromEnum(opcode))); // Given `Opcode` is not present as a tag for MIR yet + } + + /// Returns a wasm opcode from a given MIR tag. + pub fn toOpcode(self: Tag) std.wasm.Opcode { + return @as(std.wasm.Opcode, @enumFromInt(@intFromEnum(self))); + } + }; + + /// All instructions contain a 4-byte payload, which is contained within + /// this union. `Tag` determines which union tag is active, as well as + /// how to interpret the data within. + pub const Data = union { + /// Uses no additional data + tag: void, + /// Contains the result type of a block + block_type: std.wasm.BlockType, + /// Label: Each structured control instruction introduces an implicit label. + /// Labels are targets for branch instructions that reference them with + /// label indices. Unlike with other index spaces, indexing of labels + /// is relative by nesting depth, that is, label 0 refers to the + /// innermost structured control instruction enclosing the referring + /// branch instruction, while increasing indices refer to those farther + /// out. Consequently, labels can only be referenced from within the + /// associated structured control instruction. + label: u32, + /// Local: The index space for locals is only accessible inside a function and + /// includes the parameters of that function, which precede the local + /// variables. + local: u32, + /// A 32-bit immediate value. + imm32: i32, + /// A 32-bit float value + float32: f32, + /// Index into `extra`. Meaning of what can be found there is context-dependent. + payload: u32, + + ip_index: InternPool.Index, + nav_index: InternPool.Nav.Index, + intrinsic: Intrinsic, + + comptime { + switch (builtin.mode) { + .Debug, .ReleaseSafe => {}, + .ReleaseFast, .ReleaseSmall => assert(@sizeOf(Data) == 4), + } + } + }; +}; + +pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { + mir.instructions.deinit(gpa); + gpa.free(mir.extra); + gpa.free(mir.locals); + mir.uavs.deinit(gpa); + mir.indirect_function_set.deinit(gpa); + mir.func_tys.deinit(gpa); + mir.* = undefined; +} + +pub fn lower(mir: *const Mir, wasm: *Wasm, code: *std.ArrayListUnmanaged(u8)) std.mem.Allocator.Error!void { + const gpa = wasm.base.comp.gpa; + + // Write the locals in the prologue of the function body. + try code.ensureUnusedCapacity(gpa, 5 + mir.locals.len * 6 + 38); + + var w: std.Io.Writer = .fixed(code.unusedCapacitySlice()); + + w.writeLeb128(@as(u32, @intCast(mir.locals.len))) catch unreachable; + + for (mir.locals) |local| { + w.writeLeb128(@as(u32, 1)) catch unreachable; + w.writeByte(@intFromEnum(local)) catch unreachable; + } + + // Stack management section of function prologue. + const stack_alignment = mir.prologue.flags.stack_alignment; + if (stack_alignment.toByteUnits()) |align_bytes| { + const sp_global: Wasm.GlobalIndex = .stack_pointer; + // load stack pointer + w.writeByte(@intFromEnum(std.wasm.Opcode.global_get)) catch unreachable; + w.writeUleb128(@intFromEnum(sp_global)) catch unreachable; + // store stack pointer so we can restore it when we return from the function + w.writeByte(@intFromEnum(std.wasm.Opcode.local_tee)) catch unreachable; + w.writeUleb128(mir.prologue.sp_local) catch unreachable; + // get the total stack size + const aligned_stack: i32 = @intCast(stack_alignment.forward(mir.prologue.stack_size)); + w.writeByte(@intFromEnum(std.wasm.Opcode.i32_const)) catch unreachable; + w.writeSleb128(aligned_stack) catch unreachable; + // subtract it from the current stack pointer + w.writeByte(@intFromEnum(std.wasm.Opcode.i32_sub)) catch unreachable; + // Get negative stack alignment + const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1; + w.writeByte(@intFromEnum(std.wasm.Opcode.i32_const)) catch unreachable; + w.writeSleb128(neg_stack_align) catch unreachable; + // Bitwise-and the value to get the new stack pointer to ensure the + // pointers are aligned with the abi alignment. + w.writeByte(@intFromEnum(std.wasm.Opcode.i32_and)) catch unreachable; + // The bottom will be used to calculate all stack pointer offsets. + w.writeByte(@intFromEnum(std.wasm.Opcode.local_tee)) catch unreachable; + w.writeUleb128(mir.prologue.bottom_stack_local) catch unreachable; + // Store the current stack pointer value into the global stack pointer so other function calls will + // start from this value instead and not overwrite the current stack. + w.writeByte(@intFromEnum(std.wasm.Opcode.global_set)) catch unreachable; + w.writeUleb128(@intFromEnum(sp_global)) catch unreachable; + } + + code.items.len += w.end; + + var emit: Emit = .{ + .mir = mir.*, + .wasm = wasm, + .code = code, + }; + try emit.lowerToCode(); +} + +pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data: T, end: usize } { + const fields = std.meta.fields(T); + var i: usize = index; + var result: T = undefined; + inline for (fields) |field| { + @field(result, field.name) = switch (field.type) { + u32 => self.extra[i], + i32 => @bitCast(self.extra[i]), + Wasm.UavsObjIndex, + Wasm.UavsExeIndex, + InternPool.Nav.Index, + InternPool.Index, + => @enumFromInt(self.extra[i]), + else => |field_type| @compileError("Unsupported field type " ++ @typeName(field_type)), + }; + i += 1; + } + + return .{ .data = result, .end = i }; +} + +pub const JumpTable = struct { + /// Length of the jump table and the amount of entries it contains (includes default) + length: u32, +}; + +pub const Imm64 = struct { + msb: u32, + lsb: u32, + + pub fn init(full: u64) Imm64 { + return .{ + .msb = @truncate(full >> 32), + .lsb = @truncate(full), + }; + } + + pub fn toInt(i: Imm64) u64 { + return (@as(u64, i.msb) << 32) | @as(u64, i.lsb); + } +}; + +pub const Float64 = struct { + msb: u32, + lsb: u32, + + pub fn init(f: f64) Float64 { + const int: u64 = @bitCast(f); + return .{ + .msb = @truncate(int >> 32), + .lsb = @truncate(int), + }; + } + + pub fn toInt(f: Float64) u64 { + return (@as(u64, f.msb) << 32) | @as(u64, f.lsb); + } +}; + +pub const MemArg = struct { + offset: u32, + alignment: u32, +}; + +pub const UavRefOff = struct { + value: InternPool.Index, + offset: i32, +}; + +pub const NavRefOff = struct { + nav_index: InternPool.Nav.Index, + offset: i32, +}; + +/// Maps a source line with wasm bytecode +pub const DbgLineColumn = struct { + line: u32, + column: u32, +}; + +/// Tag names exactly match the corresponding symbol name. +pub const Intrinsic = enum(u32) { + __addhf3, + __addtf3, + __addxf3, + __ashlti3, + __ashrti3, + __bitreversedi2, + __bitreversesi2, + __bswapdi2, + __bswapsi2, + __ceilh, + __ceilx, + __cosh, + __cosx, + __divhf3, + __divtf3, + __divti3, + __divxf3, + __eqtf2, + __eqxf2, + __exp2h, + __exp2x, + __exph, + __expx, + __extenddftf2, + __extenddfxf2, + __extendhfsf2, + __extendhftf2, + __extendhfxf2, + __extendsftf2, + __extendsfxf2, + __extendxftf2, + __fabsh, + __fabsx, + __fixdfdi, + __fixdfsi, + __fixdfti, + __fixhfdi, + __fixhfsi, + __fixhfti, + __fixsfdi, + __fixsfsi, + __fixsfti, + __fixtfdi, + __fixtfsi, + __fixtfti, + __fixunsdfdi, + __fixunsdfsi, + __fixunsdfti, + __fixunshfdi, + __fixunshfsi, + __fixunshfti, + __fixunssfdi, + __fixunssfsi, + __fixunssfti, + __fixunstfdi, + __fixunstfsi, + __fixunstfti, + __fixunsxfdi, + __fixunsxfsi, + __fixunsxfti, + __fixxfdi, + __fixxfsi, + __fixxfti, + __floatdidf, + __floatdihf, + __floatdisf, + __floatditf, + __floatdixf, + __floatsidf, + __floatsihf, + __floatsisf, + __floatsitf, + __floatsixf, + __floattidf, + __floattihf, + __floattisf, + __floattitf, + __floattixf, + __floatundidf, + __floatundihf, + __floatundisf, + __floatunditf, + __floatundixf, + __floatunsidf, + __floatunsihf, + __floatunsisf, + __floatunsitf, + __floatunsixf, + __floatuntidf, + __floatuntihf, + __floatuntisf, + __floatuntitf, + __floatuntixf, + __floorh, + __floorx, + __fmah, + __fmax, + __fmaxh, + __fmaxx, + __fminh, + __fminx, + __fmodh, + __fmodx, + __getf2, + __gexf2, + __gttf2, + __gtxf2, + __letf2, + __lexf2, + __log10h, + __log10x, + __log2h, + __log2x, + __logh, + __logx, + __lshrti3, + __lttf2, + __ltxf2, + __modti3, + __mulhf3, + __mulodi4, + __muloti4, + __multf3, + __multi3, + __mulxf3, + __netf2, + __nexf2, + __roundh, + __roundx, + __sinh, + __sinx, + __sqrth, + __sqrtx, + __subhf3, + __subtf3, + __subxf3, + __tanh, + __tanx, + __trunch, + __truncsfhf2, + __trunctfdf2, + __trunctfhf2, + __trunctfsf2, + __trunctfxf2, + __truncx, + __truncxfdf2, + __truncxfhf2, + __truncxfsf2, + __udivti3, + __umodti3, + ceilq, + cos, + cosf, + cosq, + exp, + exp2, + exp2f, + exp2q, + expf, + expq, + fabsq, + floorq, + fma, + fmaf, + fmaq, + fmax, + fmaxf, + fmaxq, + fmin, + fminf, + fminq, + fmod, + fmodf, + fmodq, + log, + log10, + log10f, + log10q, + log2, + log2f, + log2q, + logf, + logq, + roundq, + sin, + sinf, + sinq, + sqrtq, + tan, + tanf, + tanq, + truncq, +}; diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig new file mode 100644 index 0000000000000000000000000000000000000000..c62d54ccecf0964c94f99331a3c5cffc26082224 --- /dev/null +++ b/src/codegen/x86_64/CodeGen.zig @@ -0,0 +1,194725 @@ +const std = @import("std"); +const assert = std.debug.assert; +const codegen = @import("../../codegen.zig"); +const link = @import("../../link.zig"); +const log = std.log.scoped(.codegen); +const tracking_log = std.log.scoped(.tracking); +const verbose_tracking_log = std.log.scoped(.verbose_tracking); +const wip_mir_log = std.log.scoped(.wip_mir); +const Writer = std.Io.Writer; + +const Air = @import("../../Air.zig"); +const Allocator = std.mem.Allocator; +const Emit = @import("Emit.zig"); +const Lower = @import("Lower.zig"); +const Mir = @import("Mir.zig"); +const Zcu = @import("../../Zcu.zig"); +const Module = @import("../../Package/Module.zig"); +const InternPool = @import("../../InternPool.zig"); +const Type = @import("../../Type.zig"); +const Value = @import("../../Value.zig"); + +const abi = @import("abi.zig"); +const bits = @import("bits.zig"); +const encoder = @import("encoder.zig"); + +const Condition = bits.Condition; +const Immediate = bits.Immediate; +const Memory = bits.Memory; +const Register = bits.Register; +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; +const FrameIndex = bits.FrameIndex; + +const InnerError = codegen.CodeGenError || error{OutOfRegisters}; + +pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { + return comptime &.initMany(&.{ + .scalarize_mul_sat, + .scalarize_div_floor, + .scalarize_mod, + .scalarize_add_with_overflow, + .scalarize_sub_with_overflow, + .scalarize_mul_with_overflow, + .scalarize_shl_with_overflow, + .scalarize_shr, + .scalarize_shr_exact, + .scalarize_shl, + .scalarize_shl_exact, + .scalarize_shl_sat, + .scalarize_bitcast, + .scalarize_ctz, + .scalarize_popcount, + .scalarize_byte_swap, + .scalarize_bit_reverse, + .scalarize_cmp_vector, + .scalarize_cmp_vector_optimized, + .scalarize_shuffle_one, + .scalarize_shuffle_two, + .scalarize_select, + + //.unsplat_shift_rhs, + .reduce_one_elem_to_bitcast, + .splat_one_elem_to_bitcast, + + .expand_intcast_safe, + .expand_int_from_float_safe, + .expand_int_from_float_optimized_safe, + .expand_add_safe, + .expand_sub_safe, + .expand_mul_safe, + + .expand_packed_load, + .expand_packed_store, + .expand_packed_struct_field_val, + .expand_packed_aggregate_init, + }); +} + +/// Set this to `false` to uncover Sema OPV bugs. +/// https://github.com/ziglang/zig/issues/22419 +const hack_around_sema_opv_bugs = true; + +const err_ret_trace_index: Air.Inst.Index = @enumFromInt(std.math.maxInt(u32)); + +gpa: Allocator, +pt: Zcu.PerThread, +air: Air, +liveness: Air.Liveness, +target: *const std.Target, +owner: union(enum) { + nav_index: InternPool.Nav.Index, + lazy_sym: link.File.LazySymbol, +}, +inline_func: InternPool.Index, +mod: *Module, +args: []MCValue, +va_info: union { + sysv: struct { + gp_count: u32, + fp_count: u32, + overflow_arg_area: bits.FrameAddr, + reg_save_area: bits.FrameAddr, + }, + win64: struct {}, +}, +ret_mcv: InstTracking, +err_ret_trace_reg: Register, +fn_type: Type, +src_loc: Zcu.LazySrcLoc, + +eflags_inst: ?Air.Inst.Index = null, + +/// MIR Instructions +mir_instructions: std.MultiArrayList(Mir.Inst) = .empty, +/// MIR extra data +mir_extra: std.ArrayListUnmanaged(u32) = .empty, +mir_string_bytes: std.ArrayListUnmanaged(u8) = .empty, +mir_strings: std.HashMapUnmanaged( + u32, + void, + std.hash_map.StringIndexContext, + std.hash_map.default_max_load_percentage, +) = .empty, +mir_locals: std.ArrayListUnmanaged(Mir.Local) = .empty, +mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + +/// The value is an offset into the `Function` `code` from the beginning. +/// To perform the reloc, write 32-bit signed little-endian integer +/// which is a relative jump, based on the address following the reloc. +epilogue_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, +inst_tracking: InstTrackingMap = .empty, + +// Key is the block instruction +blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, + +register_manager: RegisterManager = .{}, + +/// Generation of the current scope, increments by 1 for every entered scope. +scope_generation: u32 = 0, + +frame_allocs: std.MultiArrayList(FrameAlloc) = .empty, +free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty, +frame_locs: std.MultiArrayList(Mir.FrameLoc) = .empty, + +loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { + /// The state to restore before branching. + state: State, + /// The branch target. + target: Mir.Inst.Index, +}) = .empty, +loop_switches: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { + start: u31, + len: u11, + min: Value, + else_relocs: union(enum) { + @"unreachable", + forward: std.ArrayListUnmanaged(Mir.Inst.Index), + backward: Mir.Inst.Index, + }, +}) = .empty, + +next_temp_index: Temp.Index = @enumFromInt(0), +temp_type: [Temp.Index.max]Type = undefined, + +const MaskInfo = packed struct { + kind: enum(u1) { sign, all }, + inverted: bool = false, + scalar: Memory.Size, +}; + +pub const MCValue = union(enum) { + /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. + /// TODO Look into deleting this tag and using `dead` instead, since every use + /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. + none, + /// Control flow will not allow this value to be observed. + unreach, + /// No more references to this value remain. + /// The payload is the value of scope_generation at the point where the death occurred + dead: u32, + /// The value is undefined. + undef, + /// A pointer-sized integer that fits in a register. + /// If the type is a pointer, this is the pointer address in virtual address space. + immediate: u64, + /// The value resides in the EFLAGS register. + eflags: Condition, + /// The value is in a register. + register: Register, + /// The value is split across two registers. + register_pair: [2]Register, + /// The value is split across three registers. + register_triple: [3]Register, + /// The value is split across four registers. + register_quadruple: [4]Register, + /// The value is a constant offset from the value in a register. + register_offset: bits.RegisterOffset, + /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register. + register_overflow: struct { reg: Register, eflags: Condition }, + /// The value is a bool vector stored in a vector register with a different scalar type. + register_mask: struct { reg: Register, info: MaskInfo }, + /// The value is in memory at a hard-coded address. + /// If the type is a pointer, it means the pointer address is stored at this memory location. + memory: u64, + /// The value is in memory at a constant offset from the address in a register. + indirect: bits.RegisterOffset, + indirect_load_frame: bits.FrameAddr, + /// The value stored at an offset from a frame index + /// Payload is a frame address. + load_frame: bits.FrameAddr, + /// The address of an offset from a frame index + /// Payload is a frame address. + lea_frame: bits.FrameAddr, + load_nav: InternPool.Nav.Index, + lea_nav: InternPool.Nav.Index, + load_uav: InternPool.Key.Ptr.BaseAddr.Uav, + lea_uav: InternPool.Key.Ptr.BaseAddr.Uav, + load_lazy_sym: link.File.LazySymbol, + lea_lazy_sym: link.File.LazySymbol, + load_extern_func: Mir.NullTerminatedString, + lea_extern_func: Mir.NullTerminatedString, + /// Supports integer_per_element abi + elementwise_args: packed struct { regs: u3, frame_off: i29, frame_index: FrameIndex }, + /// This indicates that we have already allocated a frame index for this instruction, + /// but it has not been spilled there yet in the current control flow. + /// Payload is a frame index. + reserved_frame: FrameIndex, + air_ref: Air.Inst.Ref, + + fn isModifiable(mcv: MCValue) bool { + return switch (mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .register_offset, + .register_mask, + .eflags, + .register_overflow, + .indirect_load_frame, + .lea_frame, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .lea_extern_func, + .load_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => false, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .memory, + .indirect, + .load_nav, + => true, + .load_frame => |frame_addr| !frame_addr.index.isNamed(), + }; + } + + // hack around linker relocation bugs + fn isBase(mcv: MCValue) bool { + return switch (mcv) { + .memory, .indirect, .load_frame => true, + else => false, + }; + } + + fn isMemory(mcv: MCValue) bool { + return switch (mcv) { + .memory, + .indirect, + .load_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => true, + else => false, + }; + } + + fn isImmediate(mcv: MCValue) bool { + return switch (mcv) { + .immediate => true, + else => false, + }; + } + + fn isRegister(mcv: MCValue) bool { + return switch (mcv) { + .register => true, + .register_offset => |reg_off| return reg_off.off == 0, + else => false, + }; + } + + fn isRegisterOffset(mcv: MCValue) bool { + return switch (mcv) { + .register, .register_offset => true, + else => false, + }; + } + + fn getReg(mcv: MCValue) ?Register { + return switch (mcv) { + .register => |reg| reg, + .register_offset, .indirect => |ro| ro.reg, + .register_overflow => |ro| ro.reg, + .register_mask => |rm| rm.reg, + else => null, + }; + } + + fn getRegs(mcv: *const MCValue) []const Register { + return switch (mcv.*) { + .register => |*reg| reg[0..1], + inline .register_pair, + .register_triple, + .register_quadruple, + => |*regs| regs, + inline .register_offset, + .indirect, + .register_overflow, + .register_mask, + => |*pl| (&pl.reg)[0..1], + else => &.{}, + }; + } + + fn getCondition(mcv: MCValue) ?Condition { + return switch (mcv) { + .eflags => |cc| cc, + .register_overflow => |reg_ov| reg_ov.eflags, + else => null, + }; + } + + fn isAddress(mcv: MCValue) bool { + return switch (mcv) { + .immediate, .register, .register_offset, .lea_frame => true, + else => false, + }; + } + + fn address(mcv: MCValue) MCValue { + return switch (mcv) { + .none => .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .register_overflow, + .register_mask, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // not in memory + .memory => |addr| .{ .immediate = addr }, + .indirect => |reg_off| switch (reg_off.off) { + 0 => .{ .register = reg_off.reg }, + else => .{ .register_offset = reg_off }, + }, + .indirect_load_frame => |frame_addr| .{ .load_frame = frame_addr }, + .load_frame => |frame_addr| .{ .lea_frame = frame_addr }, + .load_nav => |nav| .{ .lea_nav = nav }, + .load_uav => |uav| .{ .lea_uav = uav }, + .load_lazy_sym => |lazy_sym| .{ .lea_lazy_sym = lazy_sym }, + .load_extern_func => |extern_func| .{ .lea_extern_func = extern_func }, + }; + } + + fn deref(mcv: MCValue) MCValue { + return switch (mcv) { + .none => .none, + .unreach, + .dead, + .undef, + .eflags, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + .register_mask, + .memory, + .indirect, + .indirect_load_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // not dereferenceable + .immediate => |addr| .{ .memory = addr }, + .register => |reg| .{ .indirect = .{ .reg = reg } }, + .register_offset => |reg_off| .{ .indirect = reg_off }, + .load_frame => |frame_addr| .{ .indirect_load_frame = frame_addr }, + .lea_frame => |frame_addr| .{ .load_frame = frame_addr }, + .lea_nav => |nav| .{ .load_nav = nav }, + .lea_uav => |uav| .{ .load_uav = uav }, + .lea_lazy_sym => |lazy_sym| .{ .load_lazy_sym = lazy_sym }, + .lea_extern_func => |extern_func| .{ .load_extern_func = extern_func }, + }; + } + + fn offset(mcv: MCValue, off: i32) MCValue { + return switch (mcv) { + .none => .none, + .unreach, + .dead, + .undef, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // not valid + .eflags, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + .register_mask, + .memory, + .indirect, + .indirect_load_frame, + .load_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => switch (off) { + 0 => mcv, + else => unreachable, // not offsettable + }, + .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) }, + .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } }, + .register_offset => |reg_off| .{ + .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off }, + }, + .lea_frame => |frame_addr| .{ + .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, + }, + }; + } + + fn mem(mcv: MCValue, function: *CodeGen, mod_rm: Memory.Mod.Rm) !Memory { + return switch (mcv) { + .none => .{ .mod = .{ .rm = mod_rm } }, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .register_overflow, + .register_mask, + .indirect_load_frame, + .lea_frame, + .elementwise_args, + .reserved_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => unreachable, + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = mod_rm.size, + .index = mod_rm.index, + .scale = mod_rm.scale, + .disp = small_addr + mod_rm.disp, + } }, + } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } }, + .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg.toSize(.ptr, function.target) }, + .mod = .{ .rm = .{ + .size = mod_rm.size, + .index = mod_rm.index, + .scale = mod_rm.scale, + .disp = reg_off.off + mod_rm.disp, + } }, + }, + .load_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = mod_rm.size, + .index = mod_rm.index, + .scale = mod_rm.scale, + .disp = frame_addr.off + mod_rm.disp, + } }, + }, + .load_nav => |nav| .{ .base = .{ .nav = nav }, .mod = .{ .rm = mod_rm } }, + .load_uav => |uav| .{ .base = .{ .uav = uav }, .mod = .{ .rm = mod_rm } }, + .load_lazy_sym => |lazy_sym| .{ .base = .{ .lazy_sym = lazy_sym }, .mod = .{ .rm = mod_rm } }, + .load_extern_func => |extern_func| .{ .base = .{ .extern_func = extern_func }, .mod = .{ .rm = mod_rm } }, + .air_ref => |ref| (try function.resolveInst(ref)).mem(function, mod_rm), + }; + } + + pub fn format(mcv: MCValue, w: *Writer) Writer.Error!void { + switch (mcv) { + .none, .unreach, .dead, .undef => try w.print("({s})", .{@tagName(mcv)}), + .immediate => |pl| try w.print("0x{x}", .{pl}), + .memory => |pl| try w.print("[ds:0x{x}]", .{pl}), + inline .eflags, .register => |pl| try w.print("{s}", .{@tagName(pl)}), + .register_pair => |pl| try w.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), + .register_triple => |pl| try w.print("{s}:{s}:{s}", .{ + @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]), + }), + .register_quadruple => |pl| try w.print("{s}:{s}:{s}:{s}", .{ + @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]), + }), + .register_offset => |pl| try w.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }), + .register_overflow => |pl| try w.print("{s}:{s}", .{ + @tagName(pl.eflags), + @tagName(pl.reg), + }), + .register_mask => |pl| try w.print("mask({s},{f}):{c}{s}", .{ + @tagName(pl.info.kind), + pl.info.scalar, + @as(u8, if (pl.info.inverted) '!' else ' '), + @tagName(pl.reg), + }), + .indirect => |pl| try w.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), + .indirect_load_frame => |pl| try w.print("[[{f} + 0x{x}]]", .{ pl.index, pl.off }), + .load_frame => |pl| try w.print("[{f} + 0x{x}]", .{ pl.index, pl.off }), + .lea_frame => |pl| try w.print("{f} + 0x{x}", .{ pl.index, pl.off }), + .load_nav => |pl| try w.print("[nav:{d}]", .{@intFromEnum(pl)}), + .lea_nav => |pl| try w.print("nav:{d}", .{@intFromEnum(pl)}), + .load_uav => |pl| try w.print("[uav:{d}]", .{@intFromEnum(pl.val)}), + .lea_uav => |pl| try w.print("uav:{d}", .{@intFromEnum(pl.val)}), + .load_lazy_sym => |pl| try w.print("[lazy:{s}:{d}]", .{ @tagName(pl.kind), @intFromEnum(pl.ty) }), + .lea_lazy_sym => |pl| try w.print("lazy:{s}:{d}", .{ @tagName(pl.kind), @intFromEnum(pl.ty) }), + .load_extern_func => |pl| try w.print("[extern:{d}]", .{@intFromEnum(pl)}), + .lea_extern_func => |pl| try w.print("extern:{d}", .{@intFromEnum(pl)}), + .elementwise_args => |pl| try w.print("elementwise:{d}:[{f} + 0x{x}]", .{ + pl.regs, pl.frame_index, pl.frame_off, + }), + .reserved_frame => |pl| try w.print("(dead:{f})", .{pl}), + .air_ref => |pl| try w.print("(air:0x{x})", .{@intFromEnum(pl)}), + } + } +}; + +const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking); +const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking); +const InstTracking = struct { + long: MCValue, + short: MCValue, + + fn init(result: MCValue) InstTracking { + return .{ .long = switch (result) { + .none, + .unreach, + .undef, + .immediate, + .memory, + .indirect_load_frame, + .load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => result, + .dead, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .eflags, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .register_overflow, + .register_mask, + .indirect, + => .none, + }, .short = result }; + } + + fn getReg(self: InstTracking) ?Register { + return self.short.getReg(); + } + + fn getRegs(self: *const InstTracking) []const Register { + return self.short.getRegs(); + } + + fn getCondition(self: InstTracking) ?Condition { + return self.short.getCondition(); + } + + fn spill(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { + if (std.meta.eql(self.long, self.short)) return; // Already spilled + // Allocate or reuse frame index + switch (self.long) { + .none => self.long = try cg.allocRegOrMem(inst, false), + .load_frame => {}, + .lea_frame => return, + .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, + else => unreachable, + } + tracking_log.debug("spill {f} from {f} to {f}", .{ inst, self.short, self.long }); + try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{}); + for (self.short.getRegs()) |reg| if (reg.isClass(.x87)) try cg.asmRegister(.{ .f_, .free }, reg); + } + + fn reuseFrame(self: *InstTracking) void { + self.* = .init(switch (self.long) { + .none => switch (self.short) { + .dead => .none, + else => |short| short, + }, + .reserved_frame => |index| .{ .load_frame = .{ .index = index } }, + else => |long| long, + }); + } + + fn trackSpill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void { + switch (self.short) { + .register => |reg| function.register_manager.freeReg(reg), + inline .register_pair, + .register_triple, + .register_quadruple, + => |regs| for (regs) |reg| function.register_manager.freeReg(reg), + .register_offset, .indirect => |reg_off| function.register_manager.freeReg(reg_off.reg), + .register_overflow => |reg_ov| { + function.register_manager.freeReg(reg_ov.reg); + function.eflags_inst = null; + }, + .register_mask => |reg_mask| function.register_manager.freeReg(reg_mask.reg), + .eflags => function.eflags_inst = null, + else => {}, // TODO process stack allocation death + } + self.reuseFrame(); + tracking_log.debug("{f} => {f} (spilled)", .{ inst, self.* }); + } + + fn verifyMaterialize(self: InstTracking, target: InstTracking) void { + switch (self.long) { + .none, + .load_frame, + .reserved_frame, + => switch (target.long) { + .none, + .load_frame, + .reserved_frame, + => {}, + else => unreachable, + }, + .unreach, + .undef, + .immediate, + .memory, + .indirect_load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => assert(std.meta.eql(self.long, target.long)), + .dead, + .eflags, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .register_overflow, + .register_mask, + .indirect, + .elementwise_args, + .air_ref, + => unreachable, + } + } + + fn materialize( + self: *InstTracking, + function: *CodeGen, + inst: Air.Inst.Index, + target: InstTracking, + ) !void { + self.verifyMaterialize(target); + try self.materializeUnsafe(function, inst, target); + } + + fn materializeUnsafe( + self: InstTracking, + function: *CodeGen, + inst: Air.Inst.Index, + target: InstTracking, + ) !void { + const ty = function.typeOfIndex(inst); + if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame) + try function.genCopy(ty, target.long, self.short, .{}); + try function.genCopy(ty, target.short, self.short, .{}); + } + + fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void { + self.verifyMaterialize(target); + // Don't clobber reserved frame indices + self.long = if (target.long == .none) switch (self.long) { + .load_frame => |addr| .{ .reserved_frame = addr.index }, + .reserved_frame => self.long, + else => target.long, + } else target.long; + self.short = target.short; + tracking_log.debug("{f} => {f} (materialize)", .{ inst, self.* }); + } + + fn resurrect(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index, scope_generation: u32) !void { + switch (self.short) { + .dead => |die_generation| if (die_generation >= scope_generation) { + self.reuseFrame(); + try function.getValue(self.short, inst); + tracking_log.debug("{f} => {f} (resurrect)", .{ inst, self.* }); + }, + else => {}, + } + } + + fn die(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index, comptime opts: FreeOptions) !void { + if (self.short == .dead) return; + try function.freeValue(self.short, opts); + if (self.long == .none) self.long = self.short; + self.short = .{ .dead = function.scope_generation }; + tracking_log.debug("{f} => {f} (death)", .{ inst, self.* }); + } + + fn reuse( + self: *InstTracking, + function: *CodeGen, + new_inst: ?Air.Inst.Index, + old_inst: Air.Inst.Index, + ) void { + self.short = .{ .dead = function.scope_generation }; + tracking_log.debug("{?f} => {f} (reuse {f})", .{ new_inst, self.*, old_inst }); + } + + fn liveOut(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) void { + for (self.getRegs()) |reg| { + if (function.register_manager.isRegFree(reg)) { + tracking_log.debug("{f} => {f} (live-out)", .{ inst, self.* }); + continue; + } + + const index = RegisterManager.indexOfRegIntoTracked(reg).?; + const tracked_inst = function.register_manager.registers[index]; + const tracking = function.getResolvedInstValue(tracked_inst); + + // Disable death. + var found_reg = false; + var remaining_reg: Register = .none; + for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) { + assert(!found_reg); + found_reg = true; + } else { + assert(remaining_reg == .none); + remaining_reg = tracked_reg; + }; + assert(found_reg); + if (tracking.long == .none) tracking.long = tracking.short; + tracking.short = switch (remaining_reg) { + .none => .{ .dead = function.scope_generation }, + else => .{ .register = remaining_reg }, + }; + + // Perform side-effects of freeValue manually. + function.register_manager.freeReg(reg); + + tracking_log.debug("{f} => {f} (live-out {f})", .{ inst, self.*, tracked_inst }); + } + } + + pub fn format(tracking: InstTracking, bw: *Writer) Writer.Error!void { + if (!std.meta.eql(tracking.long, tracking.short)) try bw.print("|{f}| ", .{tracking.long}); + try bw.print("{f}", .{tracking.short}); + } +}; + +const FrameAlloc = struct { + abi_size: u31, + spill_pad: u3, + abi_align: InternPool.Alignment, + ref_count: u16, + + fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc { + return .{ + .abi_size = @intCast(alloc_abi.size), + .spill_pad = alloc_abi.pad, + .abi_align = alloc_abi.alignment, + .ref_count = 0, + }; + } + fn initType(ty: Type, zcu: *Zcu) FrameAlloc { + return init(.{ + .size = ty.abiSize(zcu), + .alignment = ty.abiAlignment(zcu), + }); + } + fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc { + const abi_size = ty.abiSize(zcu); + const spill_size = if (abi_size < 8) + std.math.ceilPowerOfTwoAssert(u64, abi_size) + else + std.mem.alignForward(u64, abi_size, 8); + return init(.{ + .size = spill_size, + .pad = @intCast(spill_size - abi_size), + .alignment = ty.abiAlignment(zcu).maxStrict( + .fromNonzeroByteUnits(@min(spill_size, 8)), + ), + }); + } +}; + +const StackAllocation = struct { + inst: ?Air.Inst.Index, + /// TODO do we need size? should be determined by inst.ty.abiSize(zcu) + size: u32, +}; + +const BlockData = struct { + relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + state: State, + + fn deinit(self: *BlockData, gpa: Allocator) void { + self.relocs.deinit(gpa); + self.* = undefined; + } +}; + +const CodeGen = @This(); + +pub fn generate( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + air: *const Air, + liveness: *const ?Air.Liveness, +) codegen.CodeGenError!Mir { + _ = bin_file; + const zcu = pt.zcu; + const gpa = zcu.gpa; + const ip = &zcu.intern_pool; + const func = zcu.funcInfo(func_index); + const func_zir = func.zir_body_inst.resolveFull(ip).?; + const file = zcu.fileByIndex(func_zir.file); + const fn_type: Type = .fromInterned(func.ty); + const mod = file.mod.?; + + var function: CodeGen = .{ + .gpa = gpa, + .pt = pt, + .air = air.*, + .liveness = liveness.*.?, + .target = &mod.resolved_target.result, + .mod = mod, + .owner = .{ .nav_index = func.owner_nav }, + .inline_func = func_index, + .args = undefined, // populated after `resolveCallingConventionValues` + .va_info = undefined, // populated after `resolveCallingConventionValues` + .ret_mcv = undefined, // populated after `resolveCallingConventionValues` + .err_ret_trace_reg = undefined, // populated after `resolveCallingConventionValues` + .fn_type = fn_type, + .src_loc = src_loc, + }; + defer { + function.frame_allocs.deinit(gpa); + function.free_frame_indices.deinit(gpa); + function.frame_locs.deinit(gpa); + function.loops.deinit(gpa); + function.loop_switches.deinit(gpa); + var block_it = function.blocks.valueIterator(); + while (block_it.next()) |block| block.deinit(gpa); + function.blocks.deinit(gpa); + function.inst_tracking.deinit(gpa); + function.epilogue_relocs.deinit(gpa); + function.mir_instructions.deinit(gpa); + function.mir_string_bytes.deinit(gpa); + function.mir_strings.deinit(gpa); + function.mir_locals.deinit(gpa); + function.mir_extra.deinit(gpa); + function.mir_table.deinit(gpa); + } + try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max); + for (0..Temp.Index.max) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none)); + } + + wip_mir_log.debug("{f}:", .{fmtNav(func.owner_nav, ip)}); + + try function.frame_allocs.resize(gpa, FrameIndex.named_count); + function.frame_allocs.set( + @intFromEnum(FrameIndex.stack_frame), + .init(.{ .size = 0, .alignment = .@"1" }), + ); + function.frame_allocs.set( + @intFromEnum(FrameIndex.call_frame), + .init(.{ .size = 0, .alignment = .@"1" }), + ); + + const fn_info = zcu.typeToFunc(fn_type).?; + var call_info = function.resolveCallingConventionValues(fn_info, &.{}, .args_frame) catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + else => |e| return e, + }; + defer call_info.deinit(&function); + + function.args = call_info.args; + function.ret_mcv = call_info.return_value; + function.err_ret_trace_reg = call_info.err_ret_trace_reg; + function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), .init(.{ + .size = Type.usize.abiSize(zcu), + .alignment = Type.usize.abiAlignment(zcu).min(call_info.stack_align), + })); + function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), .init(.{ + .size = Type.usize.abiSize(zcu), + .alignment = call_info.stack_align.min( + .fromNonzeroByteUnits(function.target.stackAlignment()), + ), + })); + function.frame_allocs.set( + @intFromEnum(FrameIndex.args_frame), + .init(.{ + .size = call_info.stack_byte_count, + .alignment = call_info.stack_align, + }), + ); + function.va_info = switch (fn_info.cc) { + else => undefined, + .x86_64_sysv => .{ .sysv = .{ + .gp_count = call_info.gp_count, + .fp_count = call_info.fp_count, + .overflow_arg_area = .{ .index = .args_frame, .off = call_info.stack_byte_count }, + .reg_save_area = undefined, + } }, + .x86_64_win => .{ .win64 = .{} }, + }; + if (call_info.err_ret_trace_reg != .none) { + function.register_manager.getRegAssumeFree(call_info.err_ret_trace_reg, err_ret_trace_index); + try function.inst_tracking.putNoClobber( + gpa, + err_ret_trace_index, + .init(.{ .register = call_info.err_ret_trace_reg }), + ); + } + + function.gen(&file.zir.?, func_zir.inst, func.comptime_args, call_info.air_arg_count) catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), + else => |e| return e, + }; + + // Drop them off at the rbrace. + if (!mod.strip) _ = try function.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_line_line_column, + .data = .{ .line_column = .{ + .line = func.rbrace_line, + .column = func.rbrace_column, + } }, + }); + + var mir: Mir = .{ + .instructions = .empty, + .extra = &.{}, + .string_bytes = &.{}, + .locals = &.{}, + .table = &.{}, + .frame_locs = .empty, + }; + errdefer mir.deinit(gpa); + mir.instructions = function.mir_instructions.toOwnedSlice(); + mir.extra = try function.mir_extra.toOwnedSlice(gpa); + mir.string_bytes = try function.mir_string_bytes.toOwnedSlice(gpa); + mir.locals = try function.mir_locals.toOwnedSlice(gpa); + mir.table = try function.mir_table.toOwnedSlice(gpa); + mir.frame_locs = function.frame_locs.toOwnedSlice(); + return mir; +} + +pub fn getTmpMir(cg: *CodeGen) Mir { + return .{ + .instructions = cg.mir_instructions.slice(), + .extra = cg.mir_extra.items, + .string_bytes = cg.mir_string_bytes.items, + .locals = cg.mir_locals.items, + .table = cg.mir_table.items, + .frame_locs = cg.frame_locs.slice(), + }; +} + +pub fn generateLazy( + bin_file: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + lazy_sym: link.File.LazySymbol, + atom_index: u32, + w: *std.Io.Writer, + debug_output: link.File.DebugInfoOutput, +) codegen.CodeGenError!void { + const gpa = pt.zcu.gpa; + // This function is for generating global code, so we use the root module. + const mod = pt.zcu.comp.root_mod; + var function: CodeGen = .{ + .gpa = gpa, + .pt = pt, + .air = undefined, + .liveness = undefined, + .target = &mod.resolved_target.result, + .mod = mod, + .owner = .{ .lazy_sym = lazy_sym }, + .inline_func = undefined, + .args = undefined, + .va_info = undefined, + .ret_mcv = undefined, + .err_ret_trace_reg = undefined, + .fn_type = undefined, + .src_loc = src_loc, + }; + defer { + function.inst_tracking.deinit(gpa); + function.mir_instructions.deinit(gpa); + function.mir_string_bytes.deinit(gpa); + function.mir_strings.deinit(gpa); + function.mir_locals.deinit(gpa); + function.mir_extra.deinit(gpa); + function.mir_table.deinit(gpa); + } + try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max); + for (0..Temp.Index.max) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none)); + } + + function.genLazy(lazy_sym) catch |err| switch (err) { + error.CodegenFail => return error.CodegenFail, + error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}), + else => |e| return e, + }; + + try function.getTmpMir().emitLazy(bin_file, pt, src_loc, lazy_sym, atom_index, w, debug_output); +} + +const FormatNavData = struct { + ip: *const InternPool, + nav_index: InternPool.Nav.Index, +}; +fn formatNav(data: FormatNavData, w: *Writer) Writer.Error!void { + try w.print("{f}", .{data.ip.getNav(data.nav_index).fqn.fmt(data.ip)}); +} +fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Alt(FormatNavData, formatNav) { + return .{ .data = .{ + .ip = ip, + .nav_index = nav_index, + } }; +} + +const FormatAirData = struct { + self: *CodeGen, + inst: Air.Inst.Index, +}; +fn formatAir(data: FormatAirData, w: *Writer) Writer.Error!void { + data.self.air.writeInst(w, data.inst, data.self.pt, data.self.liveness); +} +fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Alt(FormatAirData, formatAir) { + return .{ .data = .{ .self = self, .inst = inst } }; +} + +const FormatWipMirData = struct { + self: *CodeGen, + inst: Mir.Inst.Index, +}; +fn formatWipMir(data: FormatWipMirData, w: *Writer) Writer.Error!void { + var lower: Lower = .{ + .target = data.self.target, + .allocator = data.self.gpa, + .mir = data.self.getTmpMir(), + .cc = .auto, + .src_loc = data.self.src_loc, + }; + var first = true; + for ((lower.lowerMir(data.inst) catch |err| switch (err) { + error.LowerFail => { + defer { + lower.err_msg.?.deinit(data.self.gpa); + lower.err_msg = null; + } + try w.writeAll(lower.err_msg.?.msg); + return; + }, + else => |e| { + try w.writeAll(@errorName(e)); + return; + }, + }).insts) |lowered_inst| { + if (!first) try w.writeAll("\ndebug(wip_mir): "); + try w.print(" | {f}", .{lowered_inst}); + first = false; + } + if (first) { + const ip = &data.self.pt.zcu.intern_pool; + const mir_inst = lower.mir.instructions.get(data.inst); + try w.print(" | .{s}", .{@tagName(mir_inst.ops)}); + switch (mir_inst.ops) { + else => unreachable, + .pseudo_dbg_prologue_end_none, + .pseudo_dbg_epilogue_begin_none, + .pseudo_dbg_enter_block_none, + .pseudo_dbg_leave_block_none, + .pseudo_dbg_arg_none, + .pseudo_dbg_var_args_none, + .pseudo_dbg_var_none, + .pseudo_dead_none, + => {}, + .pseudo_dbg_line_stmt_line_column, .pseudo_dbg_line_line_column => try w.print( + " {[line]d}, {[column]d}", + mir_inst.data.line_column, + ), + .pseudo_dbg_enter_inline_func, .pseudo_dbg_leave_inline_func => try w.print(" {f}", .{ + ip.getNav(ip.indexToKey(mir_inst.data.ip_index).func.owner_nav).name.fmt(ip), + }), + .pseudo_dbg_arg_i_s, .pseudo_dbg_var_i_s => try w.print(" {d}", .{ + @as(i32, @bitCast(mir_inst.data.i.i)), + }), + .pseudo_dbg_arg_i_u, .pseudo_dbg_var_i_u => try w.print(" {d}", .{ + mir_inst.data.i.i, + }), + .pseudo_dbg_arg_i_64, .pseudo_dbg_var_i_64 => try w.print(" {d}", .{ + mir_inst.data.i64, + }), + .pseudo_dbg_arg_ro, .pseudo_dbg_var_ro => { + const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ + .base = .{ .reg = mir_inst.data.ro.reg }, + .disp = mir_inst.data.ro.off, + }) }; + try w.print(" {f}", .{mem_op.fmt(.m)}); + }, + .pseudo_dbg_arg_fa, .pseudo_dbg_var_fa => { + const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ + .base = .{ .frame = mir_inst.data.fa.index }, + .disp = mir_inst.data.fa.off, + }) }; + try w.print(" {f}", .{mem_op.fmt(.m)}); + }, + .pseudo_dbg_arg_m, .pseudo_dbg_var_m => { + const mem_op: encoder.Instruction.Operand = .{ + .mem = lower.mir.extraData(Mir.Memory, mir_inst.data.x.payload).data.decode(), + }; + try w.print(" {f}", .{mem_op.fmt(.m)}); + }, + .pseudo_dbg_arg_val, .pseudo_dbg_var_val => try w.print(" {f}", .{ + Value.fromInterned(mir_inst.data.ip_index).fmtValue(data.self.pt), + }), + } + } +} +fn fmtWipMir(self: *CodeGen, inst: Mir.Inst.Index) std.fmt.Alt(FormatWipMirData, formatWipMir) { + return .{ .data = .{ .self = self, .inst = inst } }; +} + +const FormatTrackingData = struct { + self: *CodeGen, +}; +fn formatTracking(data: FormatTrackingData, w: *Writer) Writer.Error!void { + var it = data.self.inst_tracking.iterator(); + while (it.next()) |entry| try w.print("\n{f} = {f}", .{ entry.key_ptr.*, entry.value_ptr.* }); +} +fn fmtTracking(self: *CodeGen) std.fmt.Alt(FormatTrackingData, formatTracking) { + return .{ .data = .{ .self = self } }; +} + +fn addInst(self: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { + const gpa = self.gpa; + try self.mir_instructions.ensureUnusedCapacity(gpa, 1); + const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len); + self.mir_instructions.appendAssumeCapacity(inst); + if (inst.ops != .pseudo_dead_none) wip_mir_log.debug("{f}", .{self.fmtWipMir(result_index)}); + return result_index; +} + +fn addExtra(self: *CodeGen, extra: anytype) Allocator.Error!u32 { + const fields = std.meta.fields(@TypeOf(extra)); + try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len); + return self.addExtraAssumeCapacity(extra); +} + +fn addExtraAssumeCapacity(self: *CodeGen, extra: anytype) u32 { + const fields = std.meta.fields(@TypeOf(extra)); + const result: u32 = @intCast(self.mir_extra.items.len); + inline for (fields) |field| { + self.mir_extra.appendAssumeCapacity(switch (field.type) { + u32 => @field(extra, field.name), + i32, Mir.Memory.Info => @bitCast(@field(extra, field.name)), + FrameIndex => @intFromEnum(@field(extra, field.name)), + else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)), + }); + } + return result; +} + +fn addString(cg: *CodeGen, string: []const u8) Allocator.Error!Mir.NullTerminatedString { + try cg.mir_string_bytes.ensureUnusedCapacity(cg.gpa, string.len + 1); + try cg.mir_strings.ensureUnusedCapacityContext(cg.gpa, 1, .{ .bytes = &cg.mir_string_bytes }); + + const mir_string_gop = cg.mir_strings.getOrPutAssumeCapacityAdapted( + string, + std.hash_map.StringIndexAdapter{ .bytes = &cg.mir_string_bytes }, + ); + if (!mir_string_gop.found_existing) { + mir_string_gop.key_ptr.* = @intCast(cg.mir_string_bytes.items.len); + cg.mir_string_bytes.appendSliceAssumeCapacity(string); + cg.mir_string_bytes.appendAssumeCapacity(0); + } + return @enumFromInt(mir_string_gop.key_ptr.*); +} + +fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void { + return switch (ops[0]) { + .none => self.asmOpOnly(tag), + .reg => |reg0| switch (ops[1]) { + .none => self.asmRegister(tag, reg0), + .reg => |reg1| switch (ops[2]) { + .none => self.asmRegisterRegister(tag, reg0, reg1), + .reg => |reg2| switch (ops[3]) { + .none => self.asmRegisterRegisterRegister(tag, reg0, reg1, reg2), + .reg => |reg3| self.asmRegisterRegisterRegisterRegister(tag, reg0, reg1, reg2, reg3), + .imm => |imm3| self.asmRegisterRegisterRegisterImmediate(tag, reg0, reg1, reg2, imm3), + else => error.InvalidInstruction, + }, + .mem => |mem2| switch (ops[3]) { + .none => self.asmRegisterRegisterMemory(tag, reg0, reg1, mem2), + .reg => |reg3| self.asmRegisterRegisterMemoryRegister(tag, reg0, reg1, mem2, reg3), + .imm => |imm3| self.asmRegisterRegisterMemoryImmediate(tag, reg0, reg1, mem2, imm3), + else => error.InvalidInstruction, + }, + .imm => |imm2| switch (ops[3]) { + .none => self.asmRegisterRegisterImmediate(tag, reg0, reg1, imm2), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .mem => |mem1| switch (ops[2]) { + .none => self.asmRegisterMemory(tag, reg0, mem1), + .reg => |reg2| switch (ops[3]) { + .none => self.asmRegisterMemoryRegister(tag, reg0, mem1, reg2), + else => error.InvalidInstruction, + }, + .imm => |imm2| switch (ops[3]) { + .none => self.asmRegisterMemoryImmediate(tag, reg0, mem1, imm2), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .imm => |imm1| switch (ops[2]) { + .none => self.asmRegisterImmediate(tag, reg0, imm1), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .mem => |mem0| switch (ops[1]) { + .none => self.asmMemory(tag, mem0), + .reg => |reg1| switch (ops[2]) { + .none => self.asmMemoryRegister(tag, mem0, reg1), + .reg => |reg2| switch (ops[3]) { + .none => self.asmMemoryRegisterRegister(tag, mem0, reg1, reg2), + else => error.InvalidInstruction, + }, + .imm => |imm2| switch (ops[3]) { + .none => self.asmMemoryRegisterImmediate(tag, mem0, reg1, imm2), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .imm => |imm1| switch (ops[2]) { + .none => self.asmMemoryImmediate(tag, mem0, imm1), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .imm => |imm0| switch (ops[1]) { + .none => self.asmImmediate(tag, imm0), + .reg => |reg1| switch (ops[2]) { + .none => self.asmImmediateRegister(tag, imm0, reg1), + else => error.InvalidInstruction, + }, + .imm => |imm1| switch (ops[2]) { + .none => self.asmImmediateImmediate(tag, imm0, imm1), + else => error.InvalidInstruction, + }, + else => error.InvalidInstruction, + }, + .inst => |inst0| switch (ops[1]) { + .none => self.asmReloc(tag, inst0), + else => error.InvalidInstruction, + }, + }; +} + +/// A `cc` of `.z_and_np` clobbers `reg2`! +fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void { + if (self.hasFeature(.cmov)) _ = try self.addInst(.{ + .tag = switch (cc) { + else => .cmov, + .z_and_np, .nz_or_p => .pseudo, + }, + .ops = switch (cc) { + else => .rr, + .z_and_np => .pseudo_cmov_z_and_np_rr, + .nz_or_p => .pseudo_cmov_nz_or_p_rr, + }, + .data = .{ .rr = .{ + .fixes = switch (cc) { + else => .fromCond(cc), + .z_and_np, .nz_or_p => ._, + }, + .r1 = reg1, + .r2 = reg2, + } }, + }) else { + const reloc = try self.asmJccReloc(cc.negate(), undefined); + try self.asmRegisterRegister(.{ ._, .mov }, reg1, reg2); + self.performReloc(reloc); + } +} + +/// A `cc` of `.z_and_np` is not supported by this encoding! +fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void { + if (self.hasFeature(.cmov)) _ = try self.addInst(.{ + .tag = switch (cc) { + else => .cmov, + .z_and_np => unreachable, + .nz_or_p => .pseudo, + }, + .ops = switch (cc) { + else => .rm, + .z_and_np => unreachable, + .nz_or_p => .pseudo_cmov_nz_or_p_rm, + }, + .data = .{ .rx = .{ + .fixes = switch (cc) { + else => .fromCond(cc), + .z_and_np => unreachable, + .nz_or_p => ._, + }, + .r1 = reg, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }) else { + const reloc = try self.asmJccReloc(cc.negate(), undefined); + try self.asmRegisterMemory(.{ ._, .mov }, reg, m); + self.performReloc(reloc); + } +} + +fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void { + _ = try self.addInst(.{ + .tag = switch (cc) { + else => .set, + .z_and_np, .nz_or_p => .pseudo, + }, + .ops = switch (cc) { + else => .r, + .z_and_np => .pseudo_set_z_and_np_r, + .nz_or_p => .pseudo_set_nz_or_p_r, + }, + .data = switch (cc) { + else => .{ .r = .{ + .fixes = .fromCond(cc), + .r1 = reg, + } }, + .z_and_np, .nz_or_p => .{ .rr = .{ + .r1 = reg, + .r2 = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to8(), + } }, + }, + }); +} + +fn asmSetccMemory(self: *CodeGen, cc: Condition, m: Memory) !void { + const payload = try self.addExtra(Mir.Memory.encode(m)); + _ = try self.addInst(.{ + .tag = switch (cc) { + else => .set, + .z_and_np, .nz_or_p => .pseudo, + }, + .ops = switch (cc) { + else => .m, + .z_and_np => .pseudo_set_z_and_np_m, + .nz_or_p => .pseudo_set_nz_or_p_m, + }, + .data = switch (cc) { + else => .{ .x = .{ + .fixes = .fromCond(cc), + .payload = payload, + } }, + .z_and_np, .nz_or_p => .{ .rx = .{ + .r1 = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to8(), + .payload = payload, + } }, + }, + }); +} + +fn asmJmpReloc(self: *CodeGen, target: Mir.Inst.Index) !Mir.Inst.Index { + return self.addInst(.{ + .tag = .j, + .ops = .inst, + .data = .{ .inst = .{ + .fixes = ._mp, + .inst = target, + } }, + }); +} + +fn asmJccReloc(self: *CodeGen, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index { + return self.addInst(.{ + .tag = switch (cc) { + else => .j, + .z_and_np, .nz_or_p => .pseudo, + }, + .ops = switch (cc) { + else => .inst, + .z_and_np => .pseudo_j_z_and_np_inst, + .nz_or_p => .pseudo_j_nz_or_p_inst, + }, + .data = .{ .inst = .{ + .fixes = switch (cc) { + else => .fromCond(cc), + .z_and_np, .nz_or_p => ._, + }, + .inst = target, + } }, + }); +} + +fn asmReloc(self: *CodeGen, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .inst, + .data = .{ .inst = .{ + .fixes = tag[0], + .inst = target, + } }, + }); +} + +fn asmPlaceholder(self: *CodeGen) !Mir.Inst.Index { + return self.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dead_none, + .data = undefined, + }); +} + +fn asmOpOnly(self: *CodeGen, tag: Mir.Inst.FixedTag) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .none, + .data = .{ .none = .{ + .fixes = tag[0], + } }, + }); +} + +fn asmPseudo(self: *CodeGen, ops: Mir.Inst.Ops) !void { + assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and + std.mem.endsWith(u8, @tagName(ops), "_none")); + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = ops, + .data = undefined, + }); +} + +fn asmPseudoRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register) !void { + assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and + std.mem.endsWith(u8, @tagName(ops), "_r")); + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = ops, + .data = .{ .r = .{ .r1 = reg } }, + }); +} + +fn asmPseudoImmediate(self: *CodeGen, ops: Mir.Inst.Ops, imm: Immediate) !void { + assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and + std.mem.endsWith(u8, @tagName(ops), "_i_s")); + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = ops, + .data = .{ .i = .{ .i = @bitCast(imm.signed) } }, + }); +} + +fn asmPseudoRegisterRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void { + assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and + std.mem.endsWith(u8, @tagName(ops), "_rr")); + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = ops, + .data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } }, + }); +} + +fn asmPseudoRegisterImmediate(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void { + assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and + std.mem.endsWith(u8, @tagName(ops), "_ri_s")); + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = ops, + .data = .{ .ri = .{ .r1 = reg, .i = @bitCast(imm.signed) } }, + }); +} + +fn asmRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .r, + .data = .{ .r = .{ + .fixes = tag[0], + .r1 = reg, + } }, + }); +} + +fn asmImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = switch (imm) { + .signed => .i_s, + .unsigned => .i_u, + .nav => .nav, + .uav => .uav, + .lazy_sym => .lazy_sym, + .extern_func => .extern_func, + }, + .data = switch (imm) { + .signed, .unsigned => .{ .i = .{ + .fixes = tag[0], + .i = switch (imm) { + .signed => |s| @bitCast(s), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + } }, + .nav => |nav| switch (tag[0]) { + ._ => .{ .nav = nav }, + else => unreachable, + }, + .uav => |uav| switch (tag[0]) { + ._ => .{ .uav = uav }, + else => unreachable, + }, + .lazy_sym => |lazy_sym| switch (tag[0]) { + ._ => .{ .lazy_sym = lazy_sym }, + else => unreachable, + }, + .extern_func => |extern_func| switch (tag[0]) { + ._ => .{ .extern_func = extern_func }, + else => unreachable, + }, + }, + }); +} + +fn asmImmediateRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate, reg: Register) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .ir, + .data = .{ .ri = .{ + .fixes = tag[0], + .r1 = reg, + .i = @as(u8, switch (imm) { + .signed => |s| @bitCast(@as(i8, @intCast(s))), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }), + } }, + }); +} + +fn asmImmediateImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm1: Immediate, imm2: Immediate) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .ii, + .data = .{ .ii = .{ + .fixes = tag[0], + .i1 = switch (imm1) { + .signed => |s| @bitCast(@as(i16, @intCast(s))), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + .i2 = switch (imm2) { + .signed => |s| @bitCast(@as(i8, @intCast(s))), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + } }, + }); +} + +fn asmRegisterRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rr, + .data = .{ .rr = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + } }, + }); +} + +fn asmRegisterImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void { + const ops: Mir.Inst.Ops, const i: u32 = switch (imm) { + .signed => |s| .{ .ri_s, @bitCast(s) }, + .unsigned => |u| if (std.math.cast(u32, u)) |small| + .{ .ri_u, small } + else + .{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) }, + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }; + _ = try self.addInst(.{ + .tag = tag[1], + .ops = ops, + .data = .{ .ri = .{ + .fixes = tag[0], + .r1 = reg, + .i = i, + } }, + }); +} + +fn asmRegisterRegisterRegister( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + reg3: Register, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rrr, + .data = .{ .rrr = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .r3 = reg3, + } }, + }); +} + +fn asmRegisterRegisterRegisterRegister( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + reg3: Register, + reg4: Register, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rrrr, + .data = .{ .rrrr = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .r3 = reg3, + .r4 = reg4, + } }, + }); +} + +fn asmRegisterRegisterRegisterImmediate( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + reg3: Register, + imm: Immediate, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rrri, + .data = .{ .rrri = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .r3 = reg3, + .i = switch (imm) { + .signed => |s| @bitCast(@as(i8, @intCast(s))), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + } }, + }); +} + +fn asmRegisterRegisterImmediate( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + imm: Immediate, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = switch (imm) { + .signed => .rri_s, + .unsigned => .rri_u, + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + .data = .{ .rri = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .i = switch (imm) { + .signed => |s| @bitCast(s), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + } }, + }); +} + +fn asmRegisterRegisterMemory( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + m: Memory, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rrm, + .data = .{ .rrx = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmRegisterRegisterMemoryRegister( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + m: Memory, + reg3: Register, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rrmr, + .data = .{ .rrrx = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .r3 = reg3, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .m, + .data = .{ .x = .{ + .fixes = tag[0], + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmRegisterMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rm, + .data = .{ .rx = .{ + .fixes = tag[0], + .r1 = reg, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmRegisterMemoryRegister( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + m: Memory, + reg2: Register, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rmr, + .data = .{ .rrx = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmRegisterMemoryImmediate( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg: Register, + m: Memory, + imm: Immediate, +) !void { + if (switch (imm) { + .signed => |s| if (std.math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null, + .unsigned => |u| std.math.cast(u16, u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }) |small_imm| { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rmi, + .data = .{ .rix = .{ + .fixes = tag[0], + .r1 = reg, + .i = small_imm, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); + } else { + const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { + .signed => |s| @bitCast(s), + .unsigned => |u| @as(u32, @intCast(u)), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + } }); + assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); + _ = try self.addInst(.{ + .tag = tag[1], + .ops = switch (imm) { + .signed => .rmi_s, + .unsigned => .rmi_u, + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + .data = .{ .rx = .{ + .fixes = tag[0], + .r1 = reg, + .payload = payload, + } }, + }); + } +} + +fn asmRegisterRegisterMemoryImmediate( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + reg1: Register, + reg2: Register, + m: Memory, + imm: Immediate, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .rrmi, + .data = .{ .rrix = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .i = @intCast(imm.unsigned), + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmMemoryRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .mr, + .data = .{ .rx = .{ + .fixes = tag[0], + .r1 = reg, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmMemoryImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void { + const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { + .signed => |s| @bitCast(s), + .unsigned => |u| @intCast(u), + .nav, .uav, .lazy_sym, .extern_func => unreachable, + } }); + assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); + _ = try self.addInst(.{ + .tag = tag[1], + .ops = switch (imm) { + .signed => .mi_s, + .unsigned => .mi_u, + .nav, .uav, .lazy_sym, .extern_func => unreachable, + }, + .data = .{ .x = .{ + .fixes = tag[0], + .payload = payload, + } }, + }); +} + +fn asmMemoryRegisterRegister( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + m: Memory, + reg1: Register, + reg2: Register, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .mrr, + .data = .{ .rrx = .{ + .fixes = tag[0], + .r1 = reg1, + .r2 = reg2, + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn asmMemoryRegisterImmediate( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + m: Memory, + reg: Register, + imm: Immediate, +) !void { + _ = try self.addInst(.{ + .tag = tag[1], + .ops = .mri, + .data = .{ .rix = .{ + .fixes = tag[0], + .r1 = reg, + .i = @intCast(imm.unsigned), + .payload = try self.addExtra(Mir.Memory.encode(m)), + } }, + }); +} + +fn gen( + self: *CodeGen, + zir: *const std.zig.Zir, + func_zir_inst: std.zig.Zir.Inst.Index, + comptime_args: InternPool.Index.Slice, + air_arg_count: u32, +) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const fn_info = zcu.typeToFunc(self.fn_type).?; + if (fn_info.cc != .naked) { + try self.asmRegister(.{ ._, .push }, .rbp); + try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8)); + try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0)); + try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp); + try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp); + const backpatch_push_callee_preserved_regs = try self.asmPlaceholder(); + const backpatch_frame_align = try self.asmPlaceholder(); + const backpatch_frame_align_extra = try self.asmPlaceholder(); + const backpatch_stack_alloc = try self.asmPlaceholder(); + const backpatch_stack_alloc_extra = try self.asmPlaceholder(); + + switch (self.ret_mcv.long) { + .none, .unreach => {}, + .indirect => { + // The address where to store the return value for the caller is in a + // register which the callee is free to clobber. Therefore, we purposely + // spill it to stack immediately. + const frame_index = try self.allocFrameIndex(.initSpill(.usize, zcu)); + try self.genSetMem( + .{ .frame = frame_index }, + 0, + .usize, + self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off), + .{}, + ); + self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } }; + tracking_log.debug("spill {f} to {f}", .{ self.ret_mcv.long, frame_index }); + }, + else => unreachable, + } + + if (fn_info.is_var_args) switch (fn_info.cc) { + .x86_64_sysv => { + const info = &self.va_info.sysv; + const reg_save_area_fi = try self.allocFrameIndex(.init(.{ + .size = abi.SysV.c_abi_int_param_regs.len * 8 + + abi.SysV.c_abi_sse_param_regs.len * 16, + .alignment = .@"16", + })); + info.reg_save_area = .{ .index = reg_save_area_fi }; + + for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) |reg, reg_i| + try self.genSetMem(.{ .frame = reg_save_area_fi }, @intCast(reg_i * 8), .usize, .{ .register = reg }, .{}); + + try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count)); + const skip_sse_reloc = try self.asmJccReloc(.na, undefined); + + const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type }); + for (abi.SysV.c_abi_sse_param_regs[info.fp_count..], info.fp_count..) |reg, reg_i| + try self.genSetMem( + .{ .frame = reg_save_area_fi }, + @intCast(abi.SysV.c_abi_int_param_regs.len * 8 + reg_i * 16), + vec_2_f64, + .{ .register = reg }, + .{}, + ); + + self.performReloc(skip_sse_reloc); + }, + .x86_64_win => return self.fail("TODO implement gen var arg function for Win64", .{}), + else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}), + }; + + if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_prologue_end_none); + + try self.genMainBody(zir, func_zir_inst, comptime_args, air_arg_count); + + const epilogue = if (self.epilogue_relocs.items.len > 0) epilogue: { + var last_inst: Mir.Inst.Index = @intCast(self.mir_instructions.len - 1); + while (self.epilogue_relocs.getLastOrNull() == last_inst) { + self.epilogue_relocs.items.len -= 1; + self.mir_instructions.set(last_inst, .{ + .tag = .pseudo, + .ops = .pseudo_dead_none, + .data = undefined, + }); + last_inst -= 1; + } + for (self.epilogue_relocs.items) |epilogue_reloc| self.performReloc(epilogue_reloc); + + if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_epilogue_begin_none); + const backpatch_stack_dealloc = try self.asmPlaceholder(); + const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder(); + try self.asmRegister(.{ ._, .pop }, .rbp); + try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8)); + try self.asmOpOnly(.{ ._, .ret }); + break :epilogue .{ + .backpatch_stack_dealloc = backpatch_stack_dealloc, + .backpatch_pop_callee_preserved_regs = backpatch_pop_callee_preserved_regs, + }; + } else null; + + const frame_layout = try self.computeFrameLayout(fn_info.cc); + const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32); + const need_stack_adjust = frame_layout.stack_adjust > 0; + const need_save_reg = frame_layout.save_reg_list.count() > 0; + if (need_frame_align) { + const page_align = @as(u32, std.math.maxInt(u32)) << 12; + self.mir_instructions.set(backpatch_frame_align, .{ + .tag = .@"and", + .ops = .ri_s, + .data = .{ .ri = .{ + .r1 = .rsp, + .i = @max(frame_layout.stack_mask, page_align), + } }, + }); + if (frame_layout.stack_mask < page_align) { + self.mir_instructions.set(backpatch_frame_align_extra, .{ + .tag = .pseudo, + .ops = .pseudo_probe_align_ri_s, + .data = .{ .ri = .{ + .r1 = .rsp, + .i = ~frame_layout.stack_mask & page_align, + } }, + }); + } + } + if (need_stack_adjust) { + const page_size: u32 = 1 << 12; + if (frame_layout.stack_adjust <= page_size) { + self.mir_instructions.set(backpatch_stack_alloc, .{ + .tag = .sub, + .ops = .ri_s, + .data = .{ .ri = .{ + .r1 = .rsp, + .i = frame_layout.stack_adjust, + } }, + }); + } else if (frame_layout.stack_adjust < + page_size * Lower.pseudo_probe_adjust_unrolled_max_insts) + { + self.mir_instructions.set(backpatch_stack_alloc, .{ + .tag = .pseudo, + .ops = .pseudo_probe_adjust_unrolled_ri_s, + .data = .{ .ri = .{ + .r1 = .rsp, + .i = frame_layout.stack_adjust, + } }, + }); + } else { + const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc); + self.mir_instructions.set(backpatch_stack_alloc, .{ + .tag = .pseudo, + .ops = .pseudo_probe_adjust_setup_rri_s, + .data = .{ .rri = .{ + .r1 = .rsp, + .r2 = scratch_reg, + .i = frame_layout.stack_adjust, + } }, + }); + self.mir_instructions.set(backpatch_stack_alloc_extra, .{ + .tag = .pseudo, + .ops = .pseudo_probe_adjust_loop_rr, + .data = .{ .rr = .{ + .r1 = .rsp, + .r2 = scratch_reg, + } }, + }); + } + } + if (epilogue) |e| if (need_frame_align or need_stack_adjust) { + self.mir_instructions.set(e.backpatch_stack_dealloc, switch (-frame_layout.save_reg_list.size(self.target)) { + 0 => .{ + .tag = .mov, + .ops = .rr, + .data = .{ .rr = .{ + .r1 = .rsp, + .r2 = .rbp, + } }, + }, + else => |disp| .{ + .tag = .lea, + .ops = .rm, + .data = .{ .rx = .{ + .r1 = .rsp, + .payload = try self.addExtra(Mir.Memory.encode(.{ + .base = .{ .reg = .rbp }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = disp, + } }, + })), + } }, + }, + }); + }; + if (need_save_reg) { + self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{ + .tag = .pseudo, + .ops = .pseudo_push_reg_list, + .data = .{ .reg_list = frame_layout.save_reg_list }, + }); + if (epilogue) |e| self.mir_instructions.set(e.backpatch_pop_callee_preserved_regs, .{ + .tag = .pseudo, + .ops = .pseudo_pop_reg_list, + .data = .{ .reg_list = frame_layout.save_reg_list }, + }); + } + } else { + if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_prologue_end_none); + try self.genMainBody(zir, func_zir_inst, comptime_args, air_arg_count); + if (!self.mod.strip) try self.asmPseudo(.pseudo_dbg_epilogue_begin_none); + } +} + +fn genMainBody( + cg: *CodeGen, + zir: *const std.zig.Zir, + func_zir_inst: std.zig.Zir.Inst.Index, + comptime_args: InternPool.Index.Slice, + air_arg_count: u32, +) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const main_body = cg.air.getMainBody(); + const air_args_body = main_body[0..air_arg_count]; + try cg.genBody(air_args_body); + + if (!cg.mod.strip) { + var air_arg_index: usize = 0; + const fn_info = zcu.typeToFunc(cg.fn_type).?; + var fn_param_index: usize = 0; + var zir_param_index: usize = 0; + for (zir.getParamBody(func_zir_inst)) |zir_param_inst| { + const name = switch (zir.getParamName(zir_param_inst) orelse break) { + .empty => .none, + else => |zir_name| try cg.addString(zir.nullTerminatedString(zir_name)), + }; + defer zir_param_index += 1; + + if (comptime_args.len > 0) switch (comptime_args.get(ip)[zir_param_index]) { + .none => {}, + else => |comptime_arg| { + try cg.mir_locals.append(cg.gpa, .{ .name = name, .type = ip.typeOf(comptime_arg) }); + _ = try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_arg_val, + .data = .{ .ip_index = comptime_arg }, + }); + continue; + }, + }; + + const arg_ty = fn_info.param_types.get(ip)[fn_param_index]; + try cg.mir_locals.append(cg.gpa, .{ .name = name, .type = arg_ty }); + fn_param_index += 1; + + if (air_arg_index == air_args_body.len) { + try cg.asmPseudo(.pseudo_dbg_arg_none); + continue; + } + const air_arg_inst = air_args_body[air_arg_index]; + const air_arg_data = cg.air.instructions.items(.data)[air_arg_index].arg; + if (air_arg_data.zir_param_index != zir_param_index) { + try cg.asmPseudo(.pseudo_dbg_arg_none); + continue; + } + air_arg_index += 1; + try cg.genLocalDebugInfo( + .arg, + .fromInterned(arg_ty), + cg.getResolvedInstValue(air_arg_inst).short, + ); + } + if (fn_info.is_var_args) try cg.asmPseudo(.pseudo_dbg_var_args_none); + } + + try cg.genBody(main_body[air_arg_count..]); +} + +fn checkInvariantsAfterAirInst(cg: *CodeGen) void { + assert(!cg.register_manager.lockedRegsExist()); + + if (std.debug.runtime_safety) { + // check consistency of tracked registers + var it = cg.register_manager.free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = cg.register_manager.registers[index]; + const tracking = cg.getResolvedInstValue(tracked_inst); + for (tracking.getRegs()) |reg| { + if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; + } else unreachable; // tracked register not in use + } + } +} + +fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { + if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none); + try cg.genBody(body); + if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_leave_block_none); +} + +fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { + @setEvalBranchQuota(29_500); + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const air_tags = cg.air.instructions.items(.tag); + const air_datas = cg.air.instructions.items(.data); + + for (body) |inst| { + if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue; + wip_mir_log.debug("{f}", .{cg.fmtAir(inst)}); + verbose_tracking_log.debug("{f}", .{cg.fmtTracking()}); + + cg.reused_operands = .initEmpty(); + try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); + switch (air_tags[@intFromEnum(inst)]) { + .select => try cg.airSelect(inst), + .shuffle_one, .shuffle_two => @panic("x86_64 TODO: shuffle_one/shuffle_two"), + + .arg => try cg.airArg(inst), + .add, .add_optimized, .add_wrap => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .imm8, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .remainder_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp1d, .memsia(.src1d, .@"4", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .add, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .add, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .add, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .add, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .add, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, + .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .add, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .add, .dst0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .add, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .add, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .add, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .add, .dst0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_, .add, .src1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .add, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .add, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .add, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .add, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .add, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .add, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .add, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + switch (air_tag) { + else => unreachable, + .add, .add_optimized => {}, + .add_wrap => res[0].wrapInt(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + res[0].tracking(cg), + }), + else => |e| return e, + }, + } + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .add_safe => unreachable, + .add_sat => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"or", .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._po, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._pe, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .add, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .cmov, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._rx, .ro, .tmp1q, .dst0q, .sia(-31, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .sia(-31, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .sia(-31, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._a, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ .@"0:", ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp2q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp2q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp2q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .@"or", .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .@"or", .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .@"or", .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .@"or", .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .@"or", .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .adds, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .adds, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .adds, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .adds, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .adds, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .adds, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_32_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .addus, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .addus, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .addus, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .addus, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_b, .minu, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .adds, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .adds, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .adds, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .adds, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, + .{ .type = .vector_4_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, .p_w, .maxs, .dst0q, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_w, .mins, .dst0q, .lea(.tmp0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_w, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_w, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_16_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .addus, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .addus, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .addus, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .addus, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_w, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .umax_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .addus, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_d, .sra, .tmp2x, .src0x, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3x, .src0x, .dst0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_d, .sra, .tmp2x, .ui(31), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_d, .sra, .tmp2y, .src0y, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3y, .src0y, .dst0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_d, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_d, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_d, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .cmpeq, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .src1x, ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_d, .minu, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_d, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .cmpeq, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vp_, .xor, .dst0y, .src0y, .dst0y, ._ }, + .{ ._, .vp_d, .minu, .dst0y, .dst0y, .src1y, ._ }, + .{ ._, .vp_d, .add, .dst0y, .dst0y, .src0y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_d, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_d, .minu, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_d, .minu, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .src0x, .dst0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .src0y, .dst0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp2x, .dst0x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp2x, .dst0x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + .{ ._, .p_q, .cmpeq, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + .{ ._, .vp_q, .cmpeq, .tmp3y, .tmp3y, .tmp3y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .tmp2y, .dst0y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .src0x, .tmp2x, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .src0x, .tmp2x, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .src0y, .tmp2y, ._ }, + .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .tmp3y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .lea(.tmp0x), .tmp2x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .lea(.tmp0x), .tmp2x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .lea(.tmp0x), .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp2y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .lea(.tmp0y), .tmp2y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .src1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .add, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .@"or", .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .@"or", .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .adds, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .adds, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .adds, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i8, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .adds, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_b, .broadcast, .tmp3y, .lea(.tmp0b), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_b, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_b, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .maxs, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .mins, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .adds, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .adds, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .addus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .addus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .addus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .addus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"or", .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"or", .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .broadcast, .tmp2y, .lea(.tmp0b), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .minu, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .minu, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .minu, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, + .{ .type = .vector_8_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._q, .mov, .tmp2q, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .minu, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2b, .tmp1b, ._, ._ }, + .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2b, .tmp1b, ._, ._ }, + .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .adds, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .adds, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .adds, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_w, .broadcast, .tmp3y, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_w, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_w, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_w, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_w, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .maxs, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .mins, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .addus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .addus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .addus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .addus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u16, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .addus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"or", .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_w, .minu, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_w, .minu, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .minu, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .umax_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .addus, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .sub, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_d, .sra, .tmp6y, .tmp3y, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3y, .tmp3y, .tmp5y, ._ }, + .{ ._, .vp_, .xor, .tmp6y, .tmp6y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_ps, .blendv, .tmp3y, .tmp5y, .tmp6y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp5x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_d, .sra, .tmp6x, .tmp3x, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3x, .tmp3x, .tmp5x, ._ }, + .{ ._, .vp_, .xor, .tmp6x, .tmp6x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .v_ps, .blendv, .tmp3x, .tmp5x, .tmp6x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .add, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .sra, .tmp6x, .ui(31), ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp3x, .tmp5x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .blendv, .tmp5x, .tmp6x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp5x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smin), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_d, .broadcast, .tmp3y, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_d, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_d, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .maxs, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .mins, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp2y, .tmp1y, ._ }, + .{ ._, .vp_d, .minu, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .add, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, .vp_d, .minu, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .add, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .add, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .dword, .is = 32 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .dword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"or", .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .minu, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .minu, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .minu, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._a, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp5y, ._ }, + .{ ._, .vp_q, .add, .tmp6y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp3y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp3y, .tmp6y, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_pd, .blendv, .tmp5y, .tmp6y, .tmp5y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp5y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp5x, ._ }, + .{ ._, .vp_q, .add, .tmp6x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp3x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp6x, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .v_pd, .blendv, .tmp5x, .tmp6x, .tmp5x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp5x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .add, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp6x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._pd, .blendv, .tmp6x, .tmp5x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp6x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .add, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp3y, .tmp5y }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp2y, .tmp5y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .add, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp3x, .tmp5x }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp2x, .tmp5x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_q, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .add, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp4x, .tmp3x, .tmp5x, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp4x, .tmp2x, .tmp5x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .cmov, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._rx, .ro, .tmp4q, .tmp2q, .sia(-31, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp4y, .tmp5y, ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp4x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp4x, .tmp5x, ._ }, + .{ ._, .vp_, .@"or", .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp5x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp4y, ._ }, + .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp2y, .tmp4y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp4x, .tmp4x, .tmp4x, ._ }, + .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp2x, .tmp4x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp3x, .tmp2x, .tmp4x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp2y, .tmp4y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp2x, .tmp4x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .umax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .add, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp3x, .tmp2x, .tmp4x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._a, .cmov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp5p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .adc, .tmp6q, .leasi(.tmp3q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp4q, .@"8", .tmp5), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sia(2, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp3q, 8), .tmp5q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp5p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .adc, .tmp6q, .leasi(.tmp3q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp4q, .@"8", .tmp5), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.tmp4q), .si(0), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, + .{ .@"1:", ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .si(-1), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp6p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp7q, .leasi(.tmp3q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .leasi(.tmp4q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp5q, .@"8", .tmp6), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp6p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp5q, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .leaad(.tmp5, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.tmp5q), .si(0), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .adc, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp3, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .si(-1), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp7q, .leasi(.tmp3q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .leasi(.tmp4q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp5q, .@"8", .tmp6), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp6p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .leaa(.tmp5, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp5q, -8), .tmp1q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .sub, .sub_optimized, .sub_wrap => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .remainder_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1d, .memsia(.src1d, .@"4", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .sub, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .sub, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .sub, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .sub, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .sub, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, + .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ }, + .{ ._, ._, .sbb, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .sub, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .sub, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .sub, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .sub, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .sub, .dst0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .sub, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .sub, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .sub, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .sub, .dst0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_, .sub, .src1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .sub, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .sub, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .sub, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .sub, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, .f_p, .sub, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .sub, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .sub, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + switch (air_tag) { + else => unreachable, + .sub, .sub_optimized => {}, + .sub_wrap => res[0].wrapInt(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + res[0].tracking(cg), + }), + else => |e| return e, + }, + } + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .sub_safe => unreachable, + .sub_sat => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 0 }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .imm = 0 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .imm = 0 }, .to_mut_gpr, .none } }, + .{ .src = .{ .any, .imm8, .none } }, + }, + .dst_temps = .{ .{ .imm = 0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .bmi, null, null, null }, + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .andn, .dst0d, .src1d, .src0d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_mut_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .src1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._po, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._pe, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp1d, .dst0d, .uia(1, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(31, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .add, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .cmov, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._rx, .ro, .tmp1q, .dst0q, .sia(-31, .src0, .add_bit_size), ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .sia(-31, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .sia(-31, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .add, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._c, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ .@"0:", ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp2q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_mut_mmx, .none } }, + .{ .src = .{ .to_mmx, .to_mut_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .andn, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .andn, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_mut_sse, .none } }, + .{ .src = .{ .to_sse, .to_mut_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_mut_sse, .none } }, + .{ .src = .{ .to_sse, .to_mut_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .andn, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .{ .scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .andn, .dst0y, .src1y, .src0y, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .subs, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .subs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .subs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .subs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .subs, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .adds, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_32_i8, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .subus, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .subus, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .subus, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .subus, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .qword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .subs, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .subs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .subs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .subs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, + .{ .type = .vector_4_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .qword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, .p_w, .maxs, .dst0q, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_w, .mins, .dst0q, .lea(.tmp0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_w, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_w, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_16_i16, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .subus, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .subus, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .subus, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .subus, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_d, .sra, .tmp2x, .src0x, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3x, .dst0x, .src0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_d, .sra, .tmp2x, .ui(31), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp3x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_d, .sra, .tmp2y, .src0y, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3y, .dst0y, .src0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_d, .maxs, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_d, .mins, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_8_i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_d, .mins, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxu, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .vp_d, .sub, .dst0x, .dst0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .maxu, .src0x, .src1x, ._, ._ }, + .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxu, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .vp_d, .sub, .dst0y, .dst0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .src0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .src1x, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .{ .scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .dst0y, .src0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .src1y, ._ }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + .{ ._, .vp_q, .cmpeq, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp2x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + .{ ._, .p_q, .cmpeq, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp2x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + .{ ._, .vp_q, .cmpeq, .tmp3y, .tmp3y, .tmp3y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp2y, .tmp3y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .src1x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .src1x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .src0y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .src1y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .tmp2y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_q, .sub, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp0x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_q, .sub, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .tmp0y, .tmp0y, .tmp0y, ._ }, + .{ ._, .vp_q, .sub, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp0y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .yword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .andn, .tmp1y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .andn, .tmp1x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .andn, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .subs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .subs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .subs, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i8, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .subs, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 8 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smin), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_b, .broadcast, .tmp3y, .lea(.tmp0b), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_b, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_b, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .maxs, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .mins, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .slimit_delta_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .subs, .tmp3x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .adds, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .sub, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .ui(7), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._po, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + .{ ._, ._pe, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .subus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_b, .subus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .subus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .subs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .subs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .subs, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 16 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_w, .broadcast, .tmp3y, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_w, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_w, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_w, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_w, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .maxs, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .mins, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp3w, .tmp3w, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .word, .is = 15 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp2w, .tmp2w, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, .fast_imm16, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1w, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .subus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .subus, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .subus, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u16, .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .subus, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sub, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_d, .sra, .tmp6y, .tmp3y, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3y, .tmp5y, .tmp3y, ._ }, + .{ ._, .vp_, .xor, .tmp6y, .tmp6y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_ps, .blendv, .tmp3y, .tmp5y, .tmp6y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sub, .tmp5x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_d, .sra, .tmp6x, .tmp3x, .ui(31), ._ }, + .{ ._, .vp_d, .cmpgt, .tmp3x, .tmp5x, .tmp3x, ._ }, + .{ ._, .vp_, .xor, .tmp6x, .tmp6x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .v_ps, .blendv, .tmp3x, .tmp5x, .tmp6x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .sub, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .sra, .tmp6x, .ui(31), ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp3x, .tmp5x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .blendv, .tmp6x, .tmp5x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp6x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smin), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_d, .broadcast, .tmp3y, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .maxs, .tmp4y, .tmp4y, .tmp3y, ._ }, + .{ ._, .vp_d, .mins, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .maxs, .tmp4x, .tmp4x, .tmp3x, ._ }, + .{ ._, .vp_d, .mins, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .maxs, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .mins, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .dword, .is = 31 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._rx, .ro, .tmp3d, .tmp1d, .uia(1, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .ui(31), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp3d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(31, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .sa(.src0, .add_smax), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .maxu, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .sub, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .maxu, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_d, .sub, .tmp1x, .tmp2x, .tmp1x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .maxu, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .sub, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._c, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp5y, ._ }, + .{ ._, .vp_q, .sub, .tmp6y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp3y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp6y, .tmp3y, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp5y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_pd, .blendv, .tmp5y, .tmp6y, .tmp5y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp5y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp5x, ._ }, + .{ ._, .vp_q, .sub, .tmp6x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp3x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp6x, .tmp3x, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp5x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .v_pd, .blendv, .tmp5x, .tmp6x, .tmp5x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp5x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .sub, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp7x, .tmp6x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp7x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp7x, .tmp4x, ._, ._ }, + .{ ._, ._pd, .blendv, .tmp6x, .tmp5x, .tmp7x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp6x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp2y, .tmp2y, .tmp2y, ._ }, + .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp3y, .tmp5y }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .tmp4y, .tmp4y, .tmp2y, .tmp5y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp4y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp3x, .tmp5x }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .tmp4x, .tmp4x, .tmp2x, .tmp5x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_q, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .sub, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp4x, .tmp3x, .tmp5x, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp4x, .tmp2x, .tmp5x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .{ .multiple_scalar_exact_signed_int = .{ .of = .qword, .is = 63 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .cmov, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._rx, .ro, .tmp4q, .tmp2q, .sia(-31, .src0, .add_bit_size_rem_64), ._ }, + .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._o, .cmov, .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .sia(-31, .src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp5y, .tmp3y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp5x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp4x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_, .@"and", .tmp3x, .tmp5x, .tmp3x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .sub, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._c, .cmov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .qword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .blendv, .tmp2y, .tmp2y, .tmp1y, .tmp2y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_q, .sub, .tmp2x, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .blendv, .tmp2x, .tmp2x, .tmp1x, .tmp2x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .sub, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .blendv, .tmp2x, .tmp1x, .tmp2x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp1q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._c, .cmov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sia(2, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp3q, 8), .tmp5q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memiad(.dst0, .tmp0, .add_unaligned_size_add_elem_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sia(1, .src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-2, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, + .{ .@"1:", ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp3q), .tmp5q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._l, .sa, .tmp5q, .uia(63, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .add, .tmp5q, .tmp5q, ._, ._ }, + .{ ._, ._no, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp3, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, ._r, .sa, .tmp5q, .ui(63), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp3q, -8), .tmp5q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp5p, .sa(.src0, .sub_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sbb, .tmp6q, .leasi(.tmp3q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp4q, .@"8", .tmp5), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .mul, .mul_optimized => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const ty = cg.typeOf(bin_op.lhs); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ }, + .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ }, + .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp0x, .src1q, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .p_w, .movsxb, .tmp0x, .src1q, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp0x, .src1x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp0y, .src1x, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp0x, .src1q, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .p_w, .movzxb, .tmp0x, .src1q, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp0y, .src1x, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .tmp1y, .ui(1), ._ }, + .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp3x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .tmp1y, .ui(1), ._ }, + .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .ackusw, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .ackusw, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mull, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mull, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mull, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .mull, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mull, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ }, + .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ }, + .{ ._, ._z, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp9q, .tmp8q, .leai(.tmp2q, .tmp6), ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .leaiad(.tmp3q, .tmp6, .add_dst0_elem_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp9q, .leaia(.tmp3q, .tmp6, .add_dst0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .lead(.tmp6, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"2b", ._, ._, ._ }, + .{ .@"3:", ._, .mov, .leai(.tmp3q, .tmp4), .tmp5q, ._, ._ }, + .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_, .mul, .src1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .mul_safe => unreachable, + .mul_wrap => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const ty = cg.typeOf(bin_op.lhs); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ }, + .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ }, + .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ }, + .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp2x, .src1q, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .p_w, .movsxb, .tmp2x, .src1q, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .src1x, .src1x, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp1x), ._, ._ }, + .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp2y, .src1x, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ }, + .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp2x, .src1q, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ }, + .{ ._, .p_w, .movzxb, .tmp2x, .src1q, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp2y, .src1x, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ }, + .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movsxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mull, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mull, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mull, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .mull, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mull, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .mul, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ }, + .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ }, + .{ ._, ._z, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp9q, .tmp8q, .leai(.tmp2q, .tmp6), ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .leaiad(.tmp3q, .tmp6, .add_dst0_elem_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp9q, .leaia(.tmp3q, .tmp6, .add_dst0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .lead(.tmp6, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"2b", ._, ._, ._ }, + .{ .@"3:", ._, .mov, .leai(.tmp3q, .tmp4), .tmp5q, ._, ._ }, + .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp4d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_, .mul, .src1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + res[0].wrapInt(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + res[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .mul_sat => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .gphi } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .dst0h, ._, ._ }, + .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ }, + .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ }, + .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi, .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ }, + .{ ._, ._, .@"or", .tmp2w, .tmp0w, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi, .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ }, + .{ ._, ._, .@"or", .tmp2d, .tmp0d, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._c, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._e, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .cmov, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .andn, .tmp2q, .tmp1q, .dst0q, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .ui(1), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 1 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp9q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 65 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp9q, .tmp1q, ._, ._ }, + .{ ._, ._, .not, .tmp9q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._rd, .sh, .tmp9q, .tmp1q, .uia(65, .dst0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp9q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp3q, .tmp1q, .tmp0q, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .si(-1), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp0q, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp3q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .div_float, .div_float_optimized, .div_exact, .div_exact_optimized => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const ty = cg.typeOf(bin_op.lhs); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + (if (cg.floatBits(ty.scalarType(zcu))) |_| cg.select(&res, &.{ty}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_, .div, .src1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .divr, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .divr, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .div, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .div, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) else err: { + assert(air_tag == .div_exact); + res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err; + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .div_trunc => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const ty = cg.typeOf(bin_op.lhs); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + (if (cg.floatBits(ty.scalarType(zcu))) |_| cg.select(&res, &.{ty}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .zero, .precision = .inexact }) }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .tmp1x, .tmp1y, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunch" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .zero, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + .{ ._, ._ss, .round, .dst0x, .dst0d, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "truncf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "truncf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ss, .div, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, + .{ ._, .v_sd, .round, .dst0x, .dst0x, .dst0q, .rm(.{ .direction = .zero, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + .{ ._, ._sd, .round, .dst0x, .dst0q, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_pd, .round, .dst0y, .dst0y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .zero, .precision = .inexact }), ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sd, .div, .tmp1x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "trunc" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3p, .tmp2p, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .mem(.dst0), ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncx" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp1, .add_unaligned_size), ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp4t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5p, .tmp4p, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "truncq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) else err: { + res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err; + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .div_trunc_optimized, .div_floor_optimized => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(bits.RoundMode.Direction, switch (air_tag) { + else => unreachable, + .div_trunc_optimized => .zero, + .div_floor_optimized => .down, + })) { + else => unreachable, + inline .zero, .down => |direction| comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }) }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__trunch", + .down => "__floorh", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__trunch", + .down => "__floorh", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__trunch", + .down => "__floorh", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__trunch", + .down => "__floorh", + } } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__trunch", + .down => "__floorh", + } } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + .{ ._, ._ss, .round, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "truncf", + .down => "floorf", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "truncf", + .down => "floorf", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ss, .div, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .round, .tmp1x, .tmp1x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, + .{ ._, .v_sd, .round, .dst0x, .dst0x, .dst0q, .rm(.{ .direction = direction, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + .{ ._, ._sd, .round, .dst0x, .dst0q, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "trunc", + .down => "floor", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "trunc", + .down => "floor", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_pd, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_pd, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .round, .tmp1x, .tmp1x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "trunc", + .down => "floor", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sd, .div, .tmp1x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "trunc", + .down => "floor", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__truncx", + .down => "__floorx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__truncx", + .down => "__floorx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3p, .tmp2p, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__truncx", + .down => "__floorx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .mem(.dst0), ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__truncx", + .down => "__floorx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__truncx", + .down => "__floorx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "__truncx", + .down => "__floorx", + } } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp1, .add_unaligned_size), ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp4t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5p, .tmp4p, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "truncq", + .down => "floorq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "truncq", + .down => "floorq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "truncq", + .down => "floorq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .zero => "truncq", + .down => "floorq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .div_floor => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const ty = cg.typeOf(bin_op.lhs); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + (if (cg.intInfo(ty)) |int_info| err: switch (int_info.signedness) { + .signed => cg.select(&res, &.{ty}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 1 }, .{ .exact_signed_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .any, .any, .none } }, + }, + .dst_temps = .{ .{ .imm = 0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_mut_gphi, .none } }, + .{ .src = .{ .to_gpr, .to_mut_gphi, .none } }, + }, + .dst_temps = .{ .{ .reg = .al }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, + .{ ._, .i_, .div, .src1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .src1b, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1b, .dst0h, ._, ._ }, + .{ ._, ._, .cmp, .src1b, .sia(1, .src0, .add_smin), ._, ._ }, + .{ ._, ._, .adc, .dst0b, .si(-1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + .{ ._, ._, .@"and", .src1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1d, .tmp0d, ._, ._ }, + .{ ._, ._, .cmp, .src1w, .sia(1, .src0, .add_smin), ._, ._ }, + .{ ._, ._, .adc, .dst0d, .si(-1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1d, .tmp0d, ._, ._ }, + .{ ._, ._, .cmp, .src1w, .sia(1, .src0, .add_smin), ._, ._ }, + .{ ._, ._, .adc, .dst0d, .si(-1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .to_mut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1d, ._, ._, ._ }, + .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1d, .tmp0d, ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(1, .src0, .add_smin), ._, ._ }, + .{ ._, ._, .adc, .dst0d, .si(-1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .to_mut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .src1q, .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .src1q, .tmp0q, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .src1q, .tmp1q, ._, ._ }, + .{ ._, ._, .adc, .dst0q, .si(-1), ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q0, .mem(.src0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0q1, .memd(.src0q, 8), ._, ._ }, + .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q0, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp0q1, .tmp0q0, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q1, .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp0q1, .dst0q1, ._, ._ }, + .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp0q1, .tmp0q0, ._, ._ }, + .{ ._, ._, .sbb, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q0, .mem(.src0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0q1, .memd(.src0q, 8), ._, ._ }, + .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0q0, .tmp3q, ._, ._ }, + .{ ._, ._, .sbb, .dst0q1, .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divei4" } }, + .{ .kind = .{ .mem_of_type = .dst0 } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .memad(.src0d, .add_size, -4), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .memad(.src1d, .add_size, -4), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"1:", ._, .cmp, .memi(.tmp6q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._e, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_size_div_8), ._, ._ }, + .{ .@"2:", ._, .adc, .memsia(.dst0q, .@"8", .tmp0, .add_size), .si(-1), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"3:", ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + } }, + } }), + .unsigned => res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err, + } else cg.select(&res, &.{ty}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .div, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .down, .precision = .inexact }) }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .div, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .div, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .tmp2y, ._ }, + .{ ._, .v_, .cvtps2ph, .tmp1x, .tmp1y, .rm(.{}), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divhf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorh" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .div, .dst0x, .src0x, .src1d, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = .down, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + .{ ._, ._ss, .round, .dst0x, .dst0d, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "floorf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .div, .dst0x, .src1d, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .div, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "floorf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ss, .div, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .div, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .div, .dst0x, .src0x, .src1q, ._ }, + .{ ._, .v_sd, .round, .dst0x, .dst0x, .dst0q, .rm(.{ .direction = .down, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + .{ ._, ._sd, .round, .dst0x, .dst0q, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._sd, .div, .dst0x, .src1q, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .v_pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .div, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .round, .dst0x, .dst0x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .div, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .v_pd, .round, .dst0y, .dst0y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .div, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_pd, .round, .tmp1y, .tmp1y, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .div, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .round, .tmp1x, .tmp1x, .rm(.{ .direction = .down, .precision = .inexact }), ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sd, .div, .tmp1x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divdf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "floor" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3p, .tmp2p, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .mem(.dst0), ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp3t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4p, .tmp3p, ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_cc_abi = .win64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floorx" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .div, ._, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memia(.dst0, .tmp1, .add_unaligned_size), ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp4t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5p, .tmp4p, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } }, + .{ .type = .usize, .kind = .{ .extern_func = "floorq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } })) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .rem, .rem_optimized => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .any, .any, .none } }, + }, + .dst_temps = .{ .{ .imm = 0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .ah }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, + .{ ._, .i_, .div, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .ah }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .div, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .div, .src1w, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .edx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .edx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .div, .src1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .rdx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .rdx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._, .div, .src1q, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__umodei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .reg = .ax } }, + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .eax } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__umodei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, .x87, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, .x87, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, .x87, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, .x87, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .mod, .mod_optimized => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .any, .any, .none } }, + }, + .dst_temps = .{ .{ .imm = 0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_mut_gphi, .none } }, + .{ .src = .{ .to_gpr, .to_mut_gphi, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .gphi } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .ah }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, + .{ ._, .i_, .div, .src1b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .tmp0h, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .src1b, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1b, .dst0h, ._, ._ }, + .{ ._, ._, .cmp, .src1b, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gphi, .none } }, + .{ .src = .{ .to_gpr, .to_gphi, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .gphi } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .ah }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, + .{ ._, .i_, .div, .src1b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .dst0h, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .dst0h, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .ah }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .div, .src1b, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .src0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .src1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .src0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .src0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .src0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .dx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .div, .src1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .edx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1d, ._, ._, ._ }, + .{ ._, ._, .mov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .add, .src0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .src1d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src1d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .edx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1d, ._, ._, ._ }, + .{ ._, ._, .mov, .src0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src0d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .edx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .div, .src1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .to_mut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .rdx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .mov, .src0q, .dst0q, ._, ._ }, + .{ ._, ._, .add, .src0q, .src1q, ._, ._ }, + .{ ._, ._, .@"and", .src1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .src1q, .dst0q, ._, ._ }, + .{ ._, ._, .cmp, .src1q, .tmp0q, ._, ._ }, + .{ ._, ._a, .cmov, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .rdx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .mov, .src0q, .src1q, ._, ._ }, + .{ ._, ._, .@"and", .src0q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .src0q, .dst0q, ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .rdx }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._, .div, .src1q, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .src0q0, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .mov, .src0q1, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q0, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .src0q1, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .dst0q1, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp0q, .src0q1, ._, ._ }, + .{ ._, ._nae, .cmov, .src0q0, .tmp2q, ._, ._ }, + .{ ._, ._ae, .cmov, .tmp2q, .mem(.src1q), ._, ._ }, + .{ ._, ._, .add, .dst0q0, .tmp2q, ._, ._ }, + .{ ._, ._, .adc, .dst0q1, .src0q0, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q1, .mem(.src1q), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .src0q0, .memd(.src1q, 8), ._, ._ }, + .{ ._, ._, .mov, .src0q1, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q0, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .src0q1, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .dst0q1, ._, ._ }, + .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp0q, .src0q1, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .dst0q0, .mem(.src1q), ._, ._ }, + .{ ._, ._, .adc, .dst0q1, .src0q0, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__modei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-3, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .memad(.dst0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .cmp, .memad(.dst0q, .add_size, -16), .si(1), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3q, .memsi(.dst0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .sbb, .tmp3q, .si(0), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src1, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._ns, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__umodei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .src0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .src1x, .src1q, ._, ._ }, + .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .src0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .src1x, .src1q, ._, ._ }, + .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f32, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .mem(.tmp0d), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .mem(.tmp0d), ._, ._ }, + .{ ._, ._, .@"and", .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._ss, .mov, .src1x, .mem(.tmp0d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f32, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .mem(.tmp0d), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .mem(.tmp0d), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._ss, .mov, .src1x, .mem(.tmp0d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .v_ss, .add, .tmp1x, .tmp1x, .tmp3d, ._ }, + .{ ._, .v_, .cvtps2ph, .tmp1q, .tmp1x, .rm(.{}), ._ }, + .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .v_ss, .add, .tmp1x, .tmp1x, .tmp3d, ._ }, + .{ ._, .v_, .cvtps2ph, .tmp1q, .tmp1x, .rm(.{}), ._ }, + .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f16, .kind = .{ .reg = .dx } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp5w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .mem(.tmp6d), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp1d, .mem(.tmp6d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodh" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0x, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .mem(.tmp6d), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp6d), .tmp3x, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp1d, .mem(.tmp6d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .{ .type = .f32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, .v_ss, .add, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .{ .type = .f32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._d, .mov, .tmp0d, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp2d, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .cmp, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._d, .mov, .src1x, .tmp0d, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f32, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .mem(.tmp0d), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .cmp, .tmp3d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .mem(.tmp0d), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .{ .type = .f32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_d, .mov, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp3d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_d, .mov, .tmp2x, .tmp3d, ._, ._ }, + .{ ._, .v_ss, .add, .tmp1x, .tmp1x, .tmp2d, ._ }, + .{ .@"1:", .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .{ .type = .f32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._d, .mov, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp5d, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp3d, ._, ._ }, + .{ ._, ._, .@"and", .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .cmp, .tmp6d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._d, .mov, .tmp2x, .tmp3d, ._, ._ }, + .{ ._, ._ss, .add, .tmp1x, .tmp2d, ._, ._ }, + .{ .@"1:", ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodf" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp5d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .mem(.tmp4d), ._, ._ }, + .{ ._, ._, .cmp, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._ss, .add, .tmp1x, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ .@"1:", ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .{ .type = .f64, .kind = .{ .reg = .rcx } }, + .{ .type = .f64, .kind = .{ .reg = .rdx } }, + .{ .type = .f64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_q, .mov, .tmp0q, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .v_q, .mov, .tmp3q, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._, .cmp, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_q, .mov, .src1q, .tmp0q, ._, ._ }, + .{ ._, .v_sd, .add, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .{ .type = .f64, .kind = .{ .reg = .rcx } }, + .{ .type = .f64, .kind = .{ .reg = .rdx } }, + .{ .type = .f64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._q, .mov, .tmp0q, .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._q, .mov, .tmp3q, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._, .cmp, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._q, .mov, .src1x, .tmp0q, ._, ._ }, + .{ ._, ._sd, .add, .dst0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f64, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .{ .type = .f64, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .rdx } }, + .{ .type = .f64, .kind = .{ .reg = .rax } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .movl, .mem(.tmp0q), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._ps, .movl, .mem(.tmp2q), .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .mem(.tmp2q), ._, ._ }, + .{ ._, ._, .cmp, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .mem(.tmp2q), ._, ._, ._ }, + .{ ._, .f_, .add, .mem(.tmp0q), ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2q), ._, ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .movl, .dst0x, .mem(.tmp2q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .{ .type = .f64, .kind = .{ .reg = .rcx } }, + .{ .type = .f64, .kind = .{ .reg = .rdx } }, + .{ .type = .f64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_q, .mov, .tmp3q, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .v_q, .mov, .tmp6q, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .cmp, .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_q, .mov, .tmp2x, .tmp3q, ._, ._ }, + .{ ._, .v_sd, .add, .tmp1x, .tmp1x, .tmp2q, ._ }, + .{ .@"1:", .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .{ .type = .f64, .kind = .{ .reg = .rcx } }, + .{ .type = .f64, .kind = .{ .reg = .rdx } }, + .{ .type = .f64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sd, .mov, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._q, .mov, .tmp3q, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._q, .mov, .tmp6q, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .cmp, .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._q, .mov, .tmp2x, .tmp3q, ._, ._ }, + .{ ._, ._sd, .add, .tmp1x, .tmp2q, ._, ._ }, + .{ .@"1:", ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmod" } }, + .{ .type = .f64, .kind = .{ .reg = .rdx } }, + .{ .type = .f64, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .rax } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._ps, .movl, .mem(.tmp5q), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .mem(.tmp5q), ._, ._ }, + .{ ._, ._, .cmp, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .mem(.tmp5q), ._, ._, ._ }, + .{ ._, .f_, .add, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp5q), ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .mem(.tmp5q), ._, ._ }, + .{ .@"1:", ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .avx, .x87, .fast_imm16 }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ }, + .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .avx, .x87, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse2, .x87, .fast_imm16 }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ }, + .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse2, .x87, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse, .x87, .fast_imm16 }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ }, + .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse, .x87, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .f_, .ld, .dst0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .avx, .x87, .fast_imm16 }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ }, + .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .avx, .x87, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ }, + .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse2, .x87, .fast_imm16 }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ }, + .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse2, .x87, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ }, + .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse, .x87, .fast_imm16 }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ }, + .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_cc_abi = .sysv64, + .required_features = .{ .@"64bit", .sse, .x87, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .size = 16 * 2, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ }, + .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ }, + .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f128, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rcx } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .vp_q, .extr, .tmp3q, .dst0x, .ui(1), ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, .v_q, .mov, .tmp3q, .dst0x, ._, ._ }, + .{ ._, ._, .cmp, .tmp3q, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .src1x, .mem(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse4_1, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f128, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rcx } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .p_q, .extr, .tmp3q, .dst0x, .ui(1), ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._q, .mov, .tmp3q, .dst0x, ._, ._ }, + .{ ._, ._, .cmp, .tmp3q, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._dqa, .mov, .src1x, .mem(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f128, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rcx } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .p_d, .shuf, .src1x, .dst0x, .ui(0b11_10_11_10), ._ }, + .{ ._, ._q, .mov, .tmp3q, .src1x, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._q, .mov, .tmp3q, .dst0x, ._, ._ }, + .{ ._, ._, .cmp, .tmp3q, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._dqa, .mov, .src1x, .mem(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f128, .kind = .mem }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .mem }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .mem(.tmp0x), .src1x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp3x), .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .memd(.tmp3q, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp3q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._ps, .mova, .src1x, .mem(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rcx } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .vp_q, .extr, .tmp5q, .tmp1x, .ui(1), ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, .v_q, .mov, .tmp5q, .tmp1x, ._, ._ }, + .{ ._, ._, .cmp, .tmp5q, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse4_1, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rcx } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .p_q, .extr, .tmp5q, .tmp1x, .ui(1), ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._q, .mov, .tmp5q, .tmp1x, ._, ._ }, + .{ ._, ._, .cmp, .tmp5q, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rcx } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .tmp1x, .ui(0b11_10_11_10), ._ }, + .{ ._, ._q, .mov, .tmp5q, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._q, .mov, .tmp5q, .tmp1x, ._, ._ }, + .{ ._, ._, .cmp, .tmp5q, .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } }, + .{ .type = .f128, .kind = .{ .reg = .rdx } }, + .{ .type = .f128, .kind = .mem }, + .{ .type = .f128, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp5x), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .memd(.tmp5q, 8), ._, ._ }, + .{ ._, ._, .cmp, .mem(.tmp5q), .si(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp7d, ._, ._, ._ }, + .{ .@"1:", ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .ptr_add => |air_tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + try ops[0].toSlicePtr(cg); + var res: [1]Temp = undefined; + if (!hack_around_sema_opv_bugs or ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{ + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leaa(.src0, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 2 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"2", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 2 + 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"2", .src1), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 4 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"4", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 4 + 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"4", .src1), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .elem_size_is = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"8", .src1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .elem_size_is = 8 + 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"8", .src1), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .po2_elem_size, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, + } }, + }, .{ + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .add_src0_elem_size), ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + } else res[0] = ops[0]; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .ptr_sub => |air_tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + try ops[0].toSlicePtr(cg); + var res: [1]Temp = undefined; + if (!hack_around_sema_opv_bugs or ty_pl.ty.toType().elemType2(zcu).hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{ty_pl.ty.toType()}, &ops, comptime &.{ .{ + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leaa(.src0, .sub_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .neg, .src1p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 2 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .neg, .src1p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"2", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 2 + 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"2", .src1), ._, ._ }, + .{ ._, ._, .neg, .dst0p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 4 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .neg, .src1p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"4", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .elem_size_is = 4 + 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"4", .src1), ._, ._ }, + .{ ._, ._, .neg, .dst0p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .elem_size_is = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .neg, .src1p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leasi(.src0, .@"8", .src1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .elem_size_is = 8 + 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .leasi(.src1, .@"8", .src1), ._, ._ }, + .{ ._, ._, .neg, .dst0p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .po2_elem_size, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src1 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .src1p, .sa(.none, .add_log2_src0_elem_size), ._, ._ }, + .{ ._, ._, .neg, .src1p, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .src1), ._, ._ }, + } }, + }, .{ + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0p, .src1p, .sa(.none, .sub_src0_elem_size), ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.src0, .dst0), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + } else res[0] = ops[0]; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .max => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._l, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._b, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._l, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._b, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._l, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._b, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .maxs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._l, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._l, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .maxu, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .maxu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._b, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._b, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .maxs, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .maxs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .maxu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .subus, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .add, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .maxs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .maxu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .tmp1y, ._ }, + .{ ._, .vp_b, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, .vp_b, .blendv, .tmp1x, .tmp1x, .tmp2x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp1x, .tmp2x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0y, .tmp2y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp6y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp6y, .tmp5y, ._ }, + .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp4y, .tmp5y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp6x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp6x, .tmp5x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp4x, .tmp5x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp6x, .tmp5x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp3x, .tmp4x, .tmp6x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .cmp, .tmp1x, .dst0d, .dst0x, .vp(.unord) }, + .{ ._, .v_ss, .max, .dst0x, .tmp0d, .dst0x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .dst0x, .dst0x, .vp(.unord) }, + .{ ._, .v_ps, .max, .dst0x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .dst0y, .dst0y, .vp(.unord) }, + .{ ._, .v_ps, .max, .dst0y, .tmp0y, .dst0y, ._ }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, + .{ ._, .v_ps, .max, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cmp, .tmp0x, .src0x, .src0d, .vp(.unord) }, + .{ ._, .v_ss, .max, .dst0x, .src1x, .src0d, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ss, .max, .dst0x, .src0d, ._, ._ }, + .{ ._, ._ss, .cmp, .src0x, .src0d, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._ss, .max, .tmp0x, .src0d, ._, ._ }, + .{ ._, ._ss, .cmp, .dst0x, .src0d, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, + .{ ._, .v_ps, .max, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .max, .dst0x, .src0x, ._, ._ }, + .{ ._, ._ps, .cmp, .src0x, .src0x, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._ps, .max, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, + .{ ._, .v_ps, .max, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, + .{ ._, .v_ps, .max, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .max, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .max, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .tmp1x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cmp, .tmp0x, .src0x, .src0q, .vp(.unord) }, + .{ ._, .v_sd, .max, .dst0x, .src1x, .src0q, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._sd, .max, .dst0x, .src0q, ._, ._ }, + .{ ._, ._sd, .cmp, .src0x, .src0q, .sp(.unord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._sd, .max, .tmp0x, .src0q, ._, ._ }, + .{ ._, ._sd, .cmp, .dst0x, .src0q, .sp(.ord), ._ }, + .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmax" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, + .{ ._, .v_pd, .max, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .max, .dst0x, .src0x, ._, ._ }, + .{ ._, ._pd, .cmp, .src0x, .src0x, .sp(.unord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._pd, .max, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._pd, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, + .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, + .{ ._, .v_pd, .max, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, + .{ ._, .v_pd, .max, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_pd, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .max, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._pd, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .max, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._pd, .@"and", .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"or", .tmp1x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmax" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp0t, .tmp0t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp0t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._p, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, .f_, .xch, .src1t, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp1t, .tmp1t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp1t, .tmp2t, ._, ._ }, + .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp1t, .tmp2t, ._, ._ }, + .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp1t, .tmp2t, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._p, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._b, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, + .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, + .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .min => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .mins, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .mins, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .mins, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .minu, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .minu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .minu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .minu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .mins, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mins, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .mins, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mins, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .minu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .minu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .subus, .src0x, .src1x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .minu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .sub, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mins, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .mins, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mins, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .minu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .minu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .minu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 32 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp4x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp4x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp0x, .src1x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp1y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .tmp1x, .tmp1x, .tmp2x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_or_exclusive_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp1x, .tmp2x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .yword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp6y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp6y, ._ }, + .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp4y, .tmp5y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp6x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp6x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp4x, .tmp5x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .multiple_scalar_exact_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp3x, .tmp4x, .tmp5x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src1, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_unaligned_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src1, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .cmp, .tmp1x, .dst0x, .dst0d, .vp(.unord) }, + .{ ._, .v_ss, .min, .dst0x, .tmp0x, .dst0d, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .dst0x, .dst0x, .vp(.unord) }, + .{ ._, .v_ps, .min, .dst0x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .dst0y, .dst0y, .vp(.unord) }, + .{ ._, .v_ps, .min, .dst0y, .tmp0y, .dst0y, ._ }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, + .{ ._, .v_ps, .min, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_size), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp4w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cmp, .tmp0x, .src0x, .src0d, .vp(.unord) }, + .{ ._, .v_ss, .min, .dst0x, .src1x, .src0d, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ss, .min, .dst0x, .src0d, ._, ._ }, + .{ ._, ._ss, .cmp, .src0x, .src0d, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._ss, .min, .tmp0x, .src0d, ._, ._ }, + .{ ._, ._ss, .cmp, .dst0x, .src0d, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, + .{ ._, .v_ps, .min, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .min, .dst0x, .src0x, ._, ._ }, + .{ ._, ._ps, .cmp, .src0x, .src0x, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._ps, .min, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, + .{ ._, .v_ps, .min, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, + .{ ._, .v_ps, .min, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_ps, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .min, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .min, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .tmp1x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cmp, .tmp0x, .src0x, .src0q, .vp(.unord) }, + .{ ._, .v_sd, .min, .dst0x, .src1x, .src0q, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._sd, .min, .dst0x, .src0q, ._, ._ }, + .{ ._, ._sd, .cmp, .src0x, .src0q, .sp(.unord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._sd, .min, .tmp0x, .src0q, ._, ._ }, + .{ ._, ._sd, .cmp, .dst0x, .src0q, .sp(.ord), ._ }, + .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmin" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .tmp0x, .src0x, .src0x, .vp(.unord) }, + .{ ._, .v_pd, .min, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .src1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .xmm0 }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .xmm0 }, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .min, .dst0x, .src0x, ._, ._ }, + .{ ._, ._pd, .cmp, .src0x, .src0x, .sp(.unord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .src1x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .mova, .tmp0x, .src1x, ._, ._ }, + .{ ._, ._pd, .min, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._pd, .cmp, .dst0x, .src0x, .sp(.ord), ._ }, + .{ ._, ._pd, .@"and", .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._pd, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .tmp0y, .src0y, .src0y, .vp(.unord) }, + .{ ._, .v_pd, .min, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .src1y, .tmp0y }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp1y, .tmp1y, .vp(.unord) }, + .{ ._, .v_pd, .min, .tmp1y, .tmp2y, .tmp1y, ._ }, + .{ ._, .v_pd, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .min, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._pd, .blendv, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .min, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._pd, .@"and", .tmp3x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"or", .tmp1x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmin" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp0t, .tmp0t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp0t, .src1t, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp0t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._p, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, .f_, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp1t, .tmp1t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp1t, .tmp2t, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp1t, .tmp2t, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp1t, .tmp2t, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._p, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._b, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, + .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ }, + .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .add_with_overflow => |air_tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [2]Temp = undefined; + cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .po } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .add, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._rx, .sa, .dst0d, .dst0d, .tmp0d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .add, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._rx, .sa, .dst0q, .dst0q, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._l, .ro, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, + .{ ._, ._, .cmp, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .add, .src0q, .src1q, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._o, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_src0_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .dst1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .dst1q, ._, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .dst1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .dst1q, ._, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_pl.ty.toType().fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].withOverflow(&res[1], cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .sub_with_overflow => |air_tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [2]Temp = undefined; + cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .po } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .add, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._rx, .sa, .dst0d, .dst0d, .tmp0d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .add, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._rx, .sa, .dst0q, .dst0q, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._l, .ro, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .ref = .src1 } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .to_mut_gpr, .immut_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst1d, .dst1d, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .sub, .src0q, .src1q, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_mut_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .ref = .src1 } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .sub, .src0q, .src1q, ._, ._ }, + .{ ._, ._c, .set, .dst1b, ._, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._o, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_src0_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._c, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_pl.ty.toType().fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].withOverflow(&res[1], cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .mul_with_overflow => |air_tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [2]Temp = undefined; + cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nbe } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, .i_, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(16), ._, ._ }, + .{ ._, ._, .mov, .tmp0w, .dst0w, ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mul, .src1w, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .uia(std.math.maxInt(u16), .src0, .sub_umax), ._, ._ }, + .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(32), ._, ._ }, + .{ ._, ._, .@"or", .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._l, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .ref = .src0 } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1d, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"or", .tmp0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0q, .src0q, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .i_, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._l, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .dst0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .ref = .src0 } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .tmp0q, .dst0q, ._ }, + .{ ._, ._, .xor, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .ref = .src0 } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } }, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mul, .src1q, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp8q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._nz, .cmov, .tmp2q, .tmp8q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp5p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp1q, ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp10q, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp9q, .tmp3q, ._, ._ }, + .{ ._, ._, .mul, .tmp6q, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp5, .add_src0_size), .tmp9q, ._, ._ }, + .{ ._, ._c, .in, .tmp5p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp10q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp6q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp10q, .tmp6q, ._, ._ }, + .{ .@"2:", ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp10q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp3b, ._, ._ }, + .{ ._, ._e, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src0, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memad(.src1, .add_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .leasi(.tmp2q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8d, .tmp7d, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp8q, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._r, .rc, .tmp6b, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp0q, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .add, .tmp6b, .tmp6b, ._, ._ }, + .{ ._, ._, .sbb, .tmp8q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ .@"2:", ._o, .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp8q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp7b, ._, ._ }, + .{ .@"1:", ._r, .sh, .tmp5q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2q, .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"1:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp1q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4d, .tmp0d, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._x, .adc, .tmp8q, .tmp3q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leaid(.tmp1q, .tmp4, 8), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp4d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leasi(.tmp1q, .@"8", .tmp3), ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp8q, ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp5q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .tmp1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .lea, .tmp3p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .adc, .tmp8q, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), ._, ._ }, + .{ ._, ._, .adc, .tmp5b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp3), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp3, .add_src0_size), .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp3p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp0d, ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .si(0), ._, ._ }, + .{ ._, ._mp, .j, .@"2f", ._, ._, ._ }, + .{ .@"1:", ._, .@"or", .tmp8q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ .@"2:", ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp1q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_pl.ty.toType().fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].withOverflow(&res[1], cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .shl_with_overflow => |air_tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [2]Temp = undefined; + cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0b, .uia(8, .src0, .add_src1_sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._l, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0w, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0w, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .add_src1_sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0w, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .add_src1_sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .add_src1_sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .nbe } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .ne } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp4q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._l, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp6q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._r, .sa, .tmp6q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp2q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .nz } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_pl.ty.toType().fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].withOverflow(&res[1], cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .alloc => { + const ty = air_datas[@intFromEnum(inst)].ty; + const slot = try cg.tempInit(ty, .{ .lea_frame = .{ + .index = try cg.allocMemPtr(inst), + } }); + try slot.finish(inst, &.{}, &.{}, cg); + }, + .inferred_alloc, .inferred_alloc_comptime => unreachable, + .ret_ptr => { + const ty = air_datas[@intFromEnum(inst)].ty; + var slot = switch (cg.ret_mcv.long) { + else => unreachable, + .none => try cg.tempInit(ty, .{ .lea_frame = .{ + .index = try cg.allocMemPtr(inst), + } }), + .load_frame => slot: { + var slot = try cg.tempInit(ty, cg.ret_mcv.long); + try slot.toOffset(cg.ret_mcv.short.indirect.off, cg); + break :slot slot; + }, + }; + try slot.finish(inst, &.{}, &.{}, cg); + }, + .assembly => try cg.airAsm(inst), + .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) { + else => unreachable, + .bit_and, .bool_and => .@"and", + .bit_or, .bool_or => .@"or", + .xor => .xor, + })) { + else => unreachable, + inline .@"and", .@"or", .xor => |mir_tag| comptime &.{ .{ + .src_constraints = .{ .{ .size = .byte }, .{ .size = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .imm8, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .size = .word }, .{ .size = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .size = .dword }, .{ .size = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mm, .to_mm, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_xmm, .to_xmm, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .mem, .none } }, + .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_ymm, .to_ymm, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .mem, .none } }, + .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_ymm, .to_ymm, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_size = .yword }, .{ .multiple_size = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqu, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_size = .xword }, .{ .multiple_size = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .movu, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, mir_tag, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .movu, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, mir_tag, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_size = .qword }, .{ .multiple_size = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, mir_tag, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .shr, .shr_exact => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sa, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sh, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sa, .dst0w, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sh, .dst0w, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sh, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._rx, .sa, .dst0d, .src0d, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._rx, .sh, .dst0d, .src0d, .src1d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sh, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sa, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sh, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._rx, .sa, .dst0q, .src0q, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._rx, .sh, .dst0q, .src0q, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sa, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .sh, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leasia(.none, .@"8", .tmp0, .sub_src0_size), ._, ._ }, + .{ ._, ._, .not, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsia(.dst0, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memia(.src0q, .tmp1, .add_size), ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leai(.tmp2q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + cg.typeOf(bin_op.rhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .shl, .shl_exact => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0w, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_unsigned_int = 4 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0w, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._lx, .sh, .dst0d, .src0d, .src1d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_unsigned_int = 5 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._lx, .sh, .dst0q, .src0q, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_unsigned_int = 6 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .{ .to_reg = .cl }, .none } }, + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .cl }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .byte }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .unsigned_int = .word }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src1w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp4q, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + cg.typeOf(bin_op.rhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + switch (air_tag) { + else => unreachable, + .shl => res[0].wrapInt(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + res[0].tracking(cg), + }), + else => |e| return e, + }, + .shl_exact => {}, + } + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .shl_sat => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const lhs_ty = cg.typeOf(bin_op.lhs); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + switch (ops[1].tracking(cg).short) { + .immediate => |rhs_imm| res[0] = try cg.tempInit( + .i32, + .{ .immediate = if (rhs_imm < cg.intInfo(lhs_ty).?.bits) + rhs_imm + else + @bitCast(@as(i64, -1)) }, + ), + else => cg.select(&res, &.{.i32}, ops[1..2], comptime &.{ .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .si(1 << 16), ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + .{ ._, ._nb, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .si(1 << 16), ._, ._ }, + .{ ._, ._, .cmc, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .ecx }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(1 << 16), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .mem(.src0q), ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._na, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .ecx }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(1 << 16), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .mem(.src0q), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .sbb, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.rhs).fmt(pt), + ops[1].tracking(cg), + }), + else => |e| return e, + }, + } + if (ops[1].index != res[0].index) try ops[1].die(cg); + ops[1] = res[0]; + cg.select(&res, &.{lhs_ty}, &ops, comptime &.{ .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._l, .sa, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, + .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .src0b, ._, ._ }, + .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0b, .src0b, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, + .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src0b, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0b, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp1b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0b, .tmp0b, ._, ._ }, + .{ ._, ._na, .cmov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp1b, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0b, .tmp0b, ._, ._ }, + .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .fast_imm16, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._l, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._l, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, + .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, + .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .src0w, ._, ._ }, + .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0w, .src0w, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src0w, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0w, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .tmp0w, ._, ._ }, + .{ ._, ._na, .cmov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0w, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0w, .tmp0w, ._, ._ }, + .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, + .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0d, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._l, .sa, .dst0d, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + .{ .@"0:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, + .{ ._, ._c, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src0d, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .dst0d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0d, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0d, .tmp0d, ._, ._ }, + .{ ._, ._na, .cmov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp1d, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0d, .tmp0d, ._, ._ }, + .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, + .{ ._, ._o, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, + .{ ._, ._no, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._a, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0q, .leaad(.src0, .add_smax_shr_src1, 1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._l, .sa, .dst0q, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ .@"0:", ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._l, .sa, .dst0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._ne, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, + .{ ._, ._nz, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = -1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .dst0q, .src0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, + .{ ._, ._c, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .imm = 1 }, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src0q, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._a, .cmov, .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax_shr_src1), ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + .{ ._, ._na, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._z, .cmov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .src0q, ._, ._ }, + .{ ._, ._l, .sh, .tmp1q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + .{ ._, ._na, .cmov, .dst0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .{ .to_reg = .ecx }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .xor, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .src0q, ._, ._ }, + .{ ._, ._l, .sh, .tmp1q, .src1b, ._, ._ }, + .{ ._, ._, .cmp, .src0q, .tmp0q, ._, ._ }, + .{ ._, ._a, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp7q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 128 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp4q, .tmp4q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .sub, .tmp6d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._ld, .sh, .tmp5q, .tmp4q, .src1b, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .leasi(.tmp2q, .@"8", .tmp6), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._c, .de, .tmp6d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp6q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp4q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .tmp7q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp7q, ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp0q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smax), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .not, .tmp0q, ._, ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .slow_incdec, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .andn, .tmp5q, .tmp1q, .tmp3q, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._, .sub, .tmp5d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_signed_int = 32 }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .{ .to_reg = .cl }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src1d, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._na, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsi(.src0, .@"8", .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp2q), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._ld, .sh, .tmp4q, .tmp3q, .src1b, ._ }, + .{ ._, ._, .@"test", .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .@"or", .tmp4q, .leasi(.tmp2q, .@"8", .tmp5), ._, ._ }, + .{ ._, ._c, .de, .tmp5d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._s, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memsid(.dst0, .@"8", .tmp0, 8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .memsi(.src0q, .@"8", .tmp1), ._, ._ }, + .{ ._, ._ld, .sh, .tmp3q, .tmp5q, .src1b, ._ }, + .{ ._, ._, .mov, .leasi(.tmp2q, .@"8", .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .tmp5q, ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ .@"1:", ._l, .sh, .tmp3q, .src1b, ._, ._ }, + .{ .@"0:", ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .memad(.dst0q, .add_src0_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .tmp5q, ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ .@"2:", ._, .mov, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ .@"3:", ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .src1d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .memad(.dst0q, .add_size, -8), .tmp1q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + lhs_ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .not => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .dst0b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .dst0w, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_mm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .mmx }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ }, + .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .dst0q, .lea(.tmp0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_xmm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_xmm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_xmm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .vp(.true) }, + .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ }, + .{ ._, .v_dqu, .mov, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ }, + .{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.true) }, + .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ }, + .{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ }, + .{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.true) }, + .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ .@"0:", .v_, .xor, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memia(.dst0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, 8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, + .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .not, .memad(.dst0d, .add_size, -16), ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ }, + .{ ._, ._, .not, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, + .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .not, .memad(.dst0q, .add_size, -16), ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .not, .memad(.dst0d, .add_size, -8), ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .dword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ }, + .{ ._, ._, .not, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, + .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .memad(.dst0d, .add_size, -16), .sa(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -16), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16), .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -16 + 4), .si(0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16 + 8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .memad(.dst0d, .add_size, -8), .sa(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .memad(.src0d, .add_size, -8), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8), .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0d, .add_size, -8 + 4), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16), ._, ._, ._ }, + .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, -16 + 8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .not, .memiad(.dst0q, .tmp0, .add_size, -8), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memiad(.src0q, .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .not, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ .{ .signed_int_or_full_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_mm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .mmx }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ }, + .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .dst0q, .lea(.tmp0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_xmm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_xmm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_xmm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .vp(.true) }, + .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp2q, .leai(.tmp1q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .leai(.tmp1d, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 4), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .bitcast => try cg.airBitCast(inst), + .block => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const block = cg.air.extraData(Air.Block, ty_pl.payload); + if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none); + try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); + if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_leave_block_none); + }, + .loop => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const block = cg.air.extraData(Air.Block, ty_pl.payload); + try cg.loops.putNoClobber(cg.gpa, inst, .{ + .state = try cg.saveState(), + .target = @intCast(cg.mir_instructions.len), + }); + defer assert(cg.loops.remove(inst)); + try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); + }, + .repeat => { + const repeat = air_datas[@intFromEnum(inst)].repeat; + const loop = cg.loops.get(repeat.loop_inst).?; + try cg.restoreState(loop.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + _ = try cg.asmJmpReloc(loop.target); + }, + .br => try cg.airBr(inst), + .trap => try cg.asmOpOnly(.{ ._2, .ud }), + .breakpoint => try cg.asmOpOnly(.{ ._3, .int }), + .ret_addr => { + var slot = try cg.tempInit(.usize, .{ .load_frame = .{ + .index = .ret_addr, + } }); + while (try slot.toRegClass(true, .general_purpose, cg)) {} + try slot.finish(inst, &.{}, &.{}, cg); + }, + .frame_addr => { + const slot = try cg.tempInit(.usize, .{ .lea_frame = .{ + .index = .base_ptr, + } }); + try slot.finish(inst, &.{}, &.{}, cg); + }, + .call => try cg.airCall(inst, .auto, .{ .safety = true }), + .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }), + .call_never_tail => try cg.airCall(inst, .never_tail, .{ .safety = true }), + .call_never_inline => try cg.airCall(inst, .never_inline, .{ .safety = true }), + .clz => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .si(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._c, .in, .dst0b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0w, .src0w, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(16, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .false_deps_lzcnt_tzcnt, .lzcnt, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lzcnt, .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .sia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .signed_po2_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .signed_po2_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_po2_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0w, .src0w, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0w, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._r, .bs, .dst0w, .src0w, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0w, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0w, .src0w, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._r, .bs, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._nz, .cmov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._z, .cmov, .src0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .bs, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bs, .src0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .dst0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_po2_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .sia(-1, .src0, .add_2_bit_size), ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .xor, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .lzcnt, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .dst0d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .@"and", .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lzcnt, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .neg, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .add_src0_bit_size, -64), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .@"and", .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._r, .bs, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasiad(.dst0, .@"8", .tmp0, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .lzcnt, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, .slow_incdec, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .sia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_2_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._r, .bs, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_4_len), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, .slow_incdec }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .si(-1), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._r, .bs, .tmp2q, .tmp2q, ._, ._ }, + .{ ._, ._z, .cmov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp3b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, .slow_incdec, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._r, .bs, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp2b, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._c, .st, ._, ._, ._, ._ }, + .{ ._, ._, .sbb, .tmp2b, .tmp1b, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_8_len), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .si(-1), ._, ._ }, + .{ ._, ._r, .bs, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .sia(-1, .src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .tmp3b, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .tmp3b, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_len), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-16, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .false_deps_lzcnt_tzcnt, .lzcnt, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .lzcnt, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._, .lzcnt, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._nc, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .neg, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp3d, .leasiad(.tmp4, .@"8", .tmp2, .add_src0_bit_size, -64), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sia(-8, .none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"1:", ._, .@"and", .tmp3q, .leai(.tmp1q, .tmp2), ._, ._ }, + .{ ._, ._r, .bs, .tmp3q, .tmp3q, ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .si(-1), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .lea, .tmp3d, .leasiad(.tmp3, .@"8", .tmp2, .sub_src0_bit_size, 1), ._, ._ }, + .{ ._, ._, .neg, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_2_len), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .ctz => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .si(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._c, .in, .dst0b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .si(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi, null, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .tzcnt, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi, .false_deps_lzcnt_tzcnt, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi, null, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi, null, null, null }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ }, + .{ ._, ._, .tzcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .tzcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bsf_bsr_0_clobbers_result, null, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .si(16), ._, ._ }, + .{ ._, ._f, .bs, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .si(32), ._, ._ }, + .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"or", .src0d, .uia(1, .src0, .add_umax), ._, ._ }, + .{ ._, ._f, .bs, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, .bsf_bsr_0_clobbers_result, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, + .{ ._, ._z, .cmov, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .si(64), ._, ._ }, + .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._s, .bt, .src0q, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._f, .bs, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, .false_deps_lzcnt_tzcnt, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .tzcnt, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bsf_bsr_0_clobbers_result, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._f, .bs, .dst0q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .sa(.src0, .add_bit_size_sub_8_size), ._, ._ }, + .{ .@"0:", ._, .lea, .dst0d, .leasia(.dst0, .@"8", .tmp0, .add_8_src0_size), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .popcount => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .neg, .dst0b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .popcnt, .dst0d, .dst0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .popcnt, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, .false_deps_popcnt, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .popcnt, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .popcnt, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_or_exact_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0b, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(0x55), ._, ._ }, + .{ ._, ._, .sub, .tmp0b, .src0b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0x33), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0x33), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0x0f), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0b, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(0x55), ._, ._ }, + .{ ._, ._, .sub, .tmp0b, .src0b, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0x33), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0x33), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0x0f), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0w, .ui(0x5555), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ui(0x3333), ._, ._ }, + .{ ._, ._, .@"and", .tmp0w, .ui(0x3333), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0w, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0d, .uia(16, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0w, .ui(0x5555), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ui(0x3333), ._, ._ }, + .{ ._, ._, .@"and", .tmp0w, .ui(0x3333), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0w, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_or_exact_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0x5555), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x3333), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x3333), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0d, .uia(16, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0x5555), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x3333), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x3333), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f), ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, + .{ ._, ._, .add, .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_or_exact_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0x55555555), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x33333333), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x33333333), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f0f0f), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f0f0f), ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, + .{ ._, .i_, .mul, .dst0d, .tmp0d, .ui(0x01010101), ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(32 - 8), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0x55555555), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x33333333), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x33333333), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0x0f0f0f0f), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0x0f0f0f0f), ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .leai(.tmp0, .dst0), ._, ._ }, + .{ ._, .i_, .mul, .dst0d, .tmp0d, .ui(0x01010101), ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(32 - 8), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x55555555, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .ui(2), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x33333333, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(4), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x01010101, .none, .repeat), ._, ._ }, + .{ ._, .i_, .mul, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(64 - 8), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sh, .src0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x55555555, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .ui(2), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x33333333, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(4), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .leai(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0x01010101, .none, .repeat), ._, ._ }, + .{ ._, .i_, .mul, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(64 - 8), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, + .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ ._, ._l, .sh, .dst0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .popcnt, .dst0q, .dst0q, ._, ._ }, + .{ .@"0:", ._, .popcnt, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ }, + .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ ._, ._l, .sh, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ }, + .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ ._, ._l, .sh, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x55555555, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp2q, .ui(2), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x33333333, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(4), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x0f0f0f0f, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .leai(.tmp1, .tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0x01010101, .none, .repeat), ._, ._ }, + .{ ._, .i_, .mul, .tmp1q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(64 - 8), ._, ._ }, + .{ ._, ._, .lea, .dst0d, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .byte_swap => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._be, .mov, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .ro, .dst0w, .ui(8), ._, ._ }, + } }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._be, .mov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._be, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .dst0d, ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._be, .mov, .dst0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .dst0d, ._, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._be, .mov, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._be, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .movbe, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._be, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp2q, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp2q, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp2q, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2q, .mem(.src0b), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp3q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .tmp3q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp3q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .bit_reverse => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 1 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp1x), .ui(0) }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_b, .broadcast, .tmp1x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_b, .shuf, .tmp1x, .src0x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0b, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0b00110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0b11001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0b01010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0b10101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp1x), .ui(0) }, + .{ ._, .vp_, .unpcklbw, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, .p_w, .sra, .src0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_b, .broadcast, .tmp1x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_b, .shuf, .tmp1x, .src0x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0b, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0b00110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0b11001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0b01010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0b10101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp1x), .ui(0) }, + .{ ._, .vp_, .unpcklbw, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .src0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_b, .broadcast, .tmp1x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_b, .shuf, .tmp1x, .src0x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp1x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp1x, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0b, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0b00110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0b11001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0b, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ui(0b01010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(0b10101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ }, + .{ ._, ._r, .sh, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.dst0x), ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .lea(.dst0x), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .ro, .src0w, .ui(8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0b0000111100001111), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1111000011110000), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b0011001100110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1100110011001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b0101010101010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1010101010101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, + .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.dst0x), ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp2x, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .lea(.dst0x), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .ro, .src0w, .ui(8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0b0000111100001111), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1111000011110000), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b0011001100110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1100110011001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b0101010101010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1010101010101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + .{ ._, .vp_w, .srl, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .uia(24, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_w, .sll, .tmp2x, .sia(-8, .dst0, .add_bit_size), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.dst0x), ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .lea(.dst0x), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp2x, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .lea(.dst0x), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.dst0x), ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .movmsk, .dst0d, .src0x, ._, ._ }, + .{ ._, ._r, .sh, .dst0w, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .ro, .src0w, .ui(8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0b0000111100001111), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1111000011110000), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b0011001100110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1100110011001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b0101010101010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b1010101010101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b00_10_00_10), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } }, + .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp2y, .src0x, ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2y, .tmp2y, .lea(.dst0y), ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lea(.dst0y), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp2y, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .src0d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0b00001111000011110000111100001111), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b11110000111100001111000011110000), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b00110011001100110011001100110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b11001100110011001100110011001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b01010101010101010101010101010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b10101010101010101010101010101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + .{ ._, .vp_d, .sra, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b00_10_00_10), ._ }, + .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } }, + .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp2y, .src0x, ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2y, .tmp2y, .lea(.dst0y), ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lea(.dst0y), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp2y, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .src0d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0b00001111000011110000111100001111), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b11110000111100001111000011110000), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b00110011001100110011001100110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b11001100110011001100110011001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b01010101010101010101010101010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b10101010101010101010101010101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + .{ ._, .vp_d, .srl, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + .{ ._, .p_d, .srl, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .gfni, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp1x), .ui(0), ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_dq, .srl, .dst0x, .ui(1), ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b00_10_00_10), ._ }, + .{ ._, .p_d, .srl, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } }, + .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } }, + .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp2y, .src0x, ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2y, .tmp2y, .lea(.dst0y), ._ }, + .{ ._, ._, .lea, .dst0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lea(.dst0y), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_b, .movmsk, .dst0d, .tmp2y, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .src0d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0d, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0d, .ui(0b00001111000011110000111100001111), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b11110000111100001111000011110000), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b00110011001100110011001100110011), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b11001100110011001100110011001100), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ui(0b01010101010101010101010101010101), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ui(0b10101010101010101010101010101010), ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.dst0x), .ui(0), ._ }, + .{ ._, ._q, .mov, .dst0q, .src0x, ._, ._ }, + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .src0q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .gfni, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .tmp1x, .src0x, .lea(.dst0x), .ui(0) }, + .{ ._, .v_q, .mov, .dst0q, .tmp1x, ._, ._ }, + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .gfni, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.dst0x), .ui(0), ._ }, + .{ ._, ._q, .mov, .dst0q, .src0x, ._, ._ }, + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .src0q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + .{ ._, .vp_q, .srl, .dst0x, .dst0x, .uia(64, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + .{ ._, .p_q, .srl, .dst0x, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .gfni, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.dst0x), .ui(0), ._ }, + .{ ._, ._q, .mov, .dst0q, .src0x, ._, ._ }, + .{ ._, .b_, .swap, .dst0q, ._, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .b_, .swap, .src0q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .src0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + .{ ._, ._r, .sh, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 128 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp2x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 128 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .dst0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp2x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .gfni, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .tmp3x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .lea(.tmp2x), ._ }, + .{ ._, .v_q, .mov, .tmp2q, .tmp3x, ._, ._ }, + .{ ._, .vp_q, .extr, .tmp4q, .tmp3x, .ui(1), ._ }, + .{ ._, ._rd, .sh, .tmp2q, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._ }, + .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .ssse3, .gfni, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .lea(.tmp2x), ._, ._ }, + .{ ._, ._q, .mov, .tmp2q, .src0x, ._, ._ }, + .{ ._, .p_q, .extr, .tmp3q, .src0x, .ui(1), ._ }, + .{ ._, ._rd, .sh, .tmp2q, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._ }, + .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .gfni, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .tmp3x, .src0x, .lea(.tmp2x), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .lea(.tmp2x), ._ }, + .{ ._, .v_q, .mov, .tmp2q, .tmp3x, ._, ._ }, + .{ ._, .vp_q, .extr, .tmp4q, .tmp3x, .ui(1), ._ }, + .{ ._, ._rd, .sh, .tmp2q, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._ }, + .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp4q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .ssse3, .gfni, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, ._b, .gf2p8affineq, .src0x, .lea(.tmp2x), .ui(0), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .lea(.tmp2x), ._, ._ }, + .{ ._, ._q, .mov, .tmp2q, .src0x, ._, ._ }, + .{ ._, .p_q, .extr, .tmp3q, .src0x, .ui(1), ._ }, + .{ ._, ._rd, .sh, .tmp2q, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._ }, + .{ ._, ._, .mov, .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp3q, .uia(128, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp3q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 256 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0y, .src0y, .lea(.tmp2y), .ui(0) }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp2y), ._ }, + .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b01_00_11_10), ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_int = 256 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .dst0y, .src0y, .lea(.tmp2y), .ui(0) }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp2x), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .v_f128, .insert, .dst0y, .tmp3y, .dst0x, .ui(1) }, + } }, + }, .{ + .required_features = .{ .avx2, .gfni, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .yword, .is = 256 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp5y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ .@"0:", .v_q, .perm, .tmp6y, .lea(.tmp1y), .ui(0b01_00_11_10), ._ }, + .{ ._, .v_b, .gf2p8affineq, .tmp6y, .tmp6y, .tmp4y, .ui(0) }, + .{ ._, .vp_b, .shuf, .tmp6y, .tmp6y, .tmp5y, ._ }, + .{ ._, .v_dqu, .mov, .memi(.dst0y, .tmp0), .tmp6y, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 32), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .gfni, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp5x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp6x, .lea(.tmp1x), ._, ._ }, + .{ ._, .v_b, .gf2p8affineq, .tmp6x, .tmp6x, .tmp4x, .ui(0) }, + .{ ._, .vp_b, .shuf, .tmp6x, .tmp6x, .tmp5x, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp6x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, .b_, .swap, .tmp5q, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, .b_, .swap, .tmp5q, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._be, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp5q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp5q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp5q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp5q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .movbe, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._be, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .uia(0b00001111000011110000111100001111, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .uia(0b00110011001100110011001100110011, .none, .repeat), ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .uia(0b01010101010101010101010101010101, .none, .repeat), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp6q, .lea(.tmp1q), ._, ._ }, + .{ ._, .b_, .swap, .tmp6q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp3q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(4), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(4), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp3q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp4q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(2), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(2), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .@"and", .tmp7q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sh, .tmp6q, .ui(1), ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp6q, .tmp5q, ._, ._ }, + .{ ._, ._, .@"or", .tmp6q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._rd, .sh, .tmp6q, .tmp2q, .uia(64, .src0, .sub_bit_size_rem_64), ._ }, + .{ ._, ._, .mov, .tmp2q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp6q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .sqrt => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(un_op)}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ss, .sqrt, .dst0x, .dst0x, .dst0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .sqrt, .dst0x, .dst0x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .sqrt, .dst0y, .dst0y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .sqrt, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__sqrth" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .sqrt, .dst0x, .dst0x, .src0d, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .sqrt, .dst0x, .src0x, .src0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .sqrt, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .sqrt, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "sqrtf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .sqrt, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .sqrt, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .sqrt, .dst0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .sqrt, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .sqrt, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrtf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrtf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .sqrt, .dst0x, .dst0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .sqrt, .dst0x, .src0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .sqrt, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .sqrt, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .sqrt, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .sqrt, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .sqrt, .dst0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .sqrt, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .sqrt, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrt" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp0b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_, .sqrt, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .round => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(un_op)}, &ops, switch (air_tag) { + else => unreachable, + inline .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .round => |name| comptime &.{ .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "h" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "f" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "f" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "f" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__" ++ @tagName(name) ++ "x" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .abs => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0b, .src0b, ._, ._ }, + .{ ._, ._s, .cmov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .si(0), ._, ._ }, + .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, + .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0b, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src0w, ._, ._ }, + .{ ._, ._s, .cmov, .dst0w, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0w, .src0w, ._, ._ }, + .{ ._, ._s, .cmov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .si(0), ._, ._ }, + .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, + .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0d, .src0d, ._, ._ }, + .{ ._, ._s, .cmov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .si(0), ._, ._ }, + .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .sub, .dst0q, .src0q, ._, ._ }, + .{ ._, ._s, .cmov, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .si(0), ._, ._ }, + .{ ._, ._ge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._ns, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .neg, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sh, .tmp2b, .ui(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp3q, .tmp1q, ._, ._ }, + .{ ._, ._c, .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, .ssse3, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_mm, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .abs, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, .ssse3, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_mm, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .abs, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, .ssse3, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_mm, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .abs, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .abs, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .abs, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .abs, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .abs, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .abs, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .abs, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .abs, .dst0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .abs, .dst0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .abs, .dst0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_b, .abs, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_w, .abs, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_d, .abs, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_b, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_w, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_d, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_b, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_w, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_d, .abs, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, .ssse3, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_b, .abs, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, .ssse3, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_w, .abs, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, .ssse3, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .p_d, .abs, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._s, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._s, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1b, .tmp1b, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._s, .cmov, .tmp1w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .neg, .tmp1d, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._s, .cmov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .neg, .tmp1d, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._s, .cmov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .neg, .tmp1q, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sh, .tmp3b, .ui(1), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ }, + .{ ._, ._c, .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .abs, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .floor, .ceil, .trunc_float => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(un_op)}, &ops, switch (@as(bits.RoundMode.Direction, switch (air_tag) { + else => unreachable, + .floor => .down, + .ceil => .up, + .trunc_float => .zero, + })) { + else => unreachable, + inline .down, .up, .zero => |direction| comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .dst0d, .rm(.{ .direction = direction, .precision = .inexact }) }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorh", + .up => "__ceilh", + .zero => "__trunch", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .round, .dst0x, .dst0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .round, .dst0y, .dst0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .round, .tmp1y, .tmp1y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorh", + .up => "__ceilh", + .zero => "__trunch", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorh", + .up => "__ceilh", + .zero => "__trunch", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorh", + .up => "__ceilh", + .zero => "__trunch", + } } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorh", + .up => "__ceilh", + .zero => "__trunch", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .round, .dst0x, .dst0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .round, .dst0x, .src0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .round, .dst0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .round, .dst0x, .src0d, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorf", + .up => "ceilf", + .zero => "truncf", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .round, .dst0y, .src0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .round, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), .rm(.{ + .direction = direction, + .precision = .inexact, + }), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .round, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), .rm(.{ + .direction = direction, + .precision = .inexact, + }), ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorf", + .up => "ceilf", + .zero => "truncf", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorf", + .up => "ceilf", + .zero => "truncf", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .round, .dst0x, .dst0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .round, .dst0x, .src0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }) }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .round, .dst0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .round, .dst0x, .src0q, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floor", + .up => "ceil", + .zero => "trunc", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .round, .dst0x, .src0x, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .round, .dst0y, .src0y, .rm(.{ .direction = direction, .precision = .inexact }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .round, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), .rm(.{ + .direction = direction, + .precision = .inexact, + }), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .round, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), .rm(.{ + .direction = direction, + .precision = .inexact, + }), ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floor", + .up => "ceil", + .zero => "trunc", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floor", + .up => "ceil", + .zero => "trunc", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floor", + .up => "ceil", + .zero => "trunc", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "__floorx", + .up => "__ceilx", + .zero => "__truncx", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorq", + .up => "ceilq", + .zero => "truncq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorq", + .up => "ceilq", + .zero => "truncq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorq", + .up => "ceilq", + .zero => "truncq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = switch (direction) { + else => unreachable, + .down => "floorq", + .up => "ceilq", + .zero => "truncq", + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .neg, .neg_optimized => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(un_op)}, &ops, comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .chs, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .xor, .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .cmp_lt, + .cmp_lt_optimized, + .cmp_lte, + .cmp_lte_optimized, + .cmp_gte, + .cmp_gte_optimized, + .cmp_gt, + .cmp_gt_optimized, + => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const cmp_op = air_tag.toCmpOp().?; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + (if (cg.floatBits(cg.typeOf(bin_op.lhs))) |_| err: { + switch (cmp_op) { + else => unreachable, + .lt, .lte => {}, + .gt, .gte => std.mem.swap(Temp, &ops[0], &ops[1]), + } + break :err cg.select(&res, &.{.bool}, &ops, switch (switch (cmp_op) { + else => unreachable, + .lt, .gt => true, + .lte, .gte => false, + }) { + inline false, true => |strict| comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ }, + .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .l, + false => .le, + } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .z, + false => .nc, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + switch (strict) { + true => .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_001), ._, ._ }, + false => .{ ._, ._r, .sh, .tmp2h, .ui(1), ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_x87, .to_x87, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_x87, .to_x87, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .a, + false => .ae, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .z, + false => .nc, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + switch (strict) { + true => .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_001), ._, ._ }, + false => .{ ._, ._r, .sh, .tmp2h, .ui(1), ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_x87, .to_x87, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .z, + false => .nc, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + switch (strict) { + true => .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_001), ._, ._ }, + false => .{ ._, ._r, .sh, .tmp2h, .ui(1), ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (strict) { + true => .l, + false => .le, + } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + } }, + }); + } else err: { + res[0] = ops[0].cmpInts(cmp_op, &ops[1], cg) catch |err| break :err err; + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .cmp_eq, + .cmp_eq_optimized, + .cmp_neq, + .cmp_neq_optimized, + => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const cmp_op = air_tag.toCmpOp().?; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + const ty = cg.typeOf(bin_op.lhs); + var res: [1]Temp = undefined; + const OptInfo = struct { + deaths: [2]Air.Inst.Index, + res: [1]Temp, + state: State, + reloc: Mir.Inst.Index, + }; + var opt_info: ?OptInfo = null; + (err: switch (@as(enum { float, int }, if (cg.floatBits(ty)) |_| + .float + else if (cg.intInfo(ty)) |_| + .int + else category: { + const child_ty = ty.optionalChild(zcu); + const has_value_off: u31 = @intCast(child_ty.abiSize(zcu)); + var has_values: [2]Temp = undefined; + opt_info = @as(OptInfo, undefined); + for (&has_values, &ops, &opt_info.?.deaths) |*has_value, *op, *death| { + has_value.* = try op.read(.bool, .{ .disp = has_value_off }, cg); + const child = try op.read(child_ty, .{}, cg); + try op.die(cg); + op.* = child; + death.* = child.index; + } + cg.select( + &opt_info.?.res, + &.{.bool}, + &has_values, + switch (Condition.fromCompareOperatorUnsigned(cmp_op)) { + else => unreachable, + inline .e, .ne => |cc| comptime &.{.{ + .src_constraints = .{ .{ .size = .byte }, .{ .size = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm8, .none } }, + .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, .fromCond(cc), .set, .dst0b, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0b, .src1b, ._, ._ }, + } }, + }}, + }, + ) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + for (has_values) |has_value| for (opt_info.?.res) |opt_res| { + if (has_value.index == opt_res.index) break; + } else try has_value.die(cg); + opt_info.?.state = cg.initRetroactiveState(); + opt_info.?.state.next_temp_index = cg.next_temp_index; + var state = try cg.saveState(); + state.next_temp_index = cg.next_temp_index; + for (ops) |op| try op.die(cg); + try cg.saveRetroactiveState(&opt_info.?.state); + opt_info.?.reloc = try cg.asmJccReloc(.z, undefined); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + break :category if (cg.floatBits(child_ty)) |_| .float else .int; + })) { + .float => { + cg.select(&res, &.{.bool}, &ops, switch (switch (air_tag) { + else => unreachable, + .cmp_eq, .cmp_neq => false, + .cmp_eq_optimized, .cmp_neq_optimized => true, + }) { + inline false, true => |optimized| comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ }, + .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z, + true => .nz, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (optimized) { + false => &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2h, .si(0b1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_100), ._, ._ }, + }, + true => &.{ + .{ ._, .f_, .ld, .src1q, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_000), ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z_and_np, + true => .z, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z, + true => .nz, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (optimized) { + false => &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2h, .si(0b1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_100), ._, ._ }, + }, + true => &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_000), ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87, .none } }, + .{ .src = .{ .to_x87, .to_x87, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = switch (optimized) { + false => .z, + true => .nz, + } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (optimized) { + false => &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2h, .si(0b1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_100), ._, ._ }, + }, + true => &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp2h, .si(0b1_000_000), ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + } }, + }) catch |err| break :err err; + switch (cmp_op) { + else => unreachable, + .eq => {}, + .neq => { + const cc = &res[0].unwrap(cg).temp.tracking(cg).short.eflags; + cc.* = cc.negate(); + }, + } + }, + .int => { + switch (ty.zigTypeTag(zcu)) { + else => {}, + .@"struct", .@"union" => { + assert(ty.containerLayout(zcu) == .@"packed"); + for (&ops) |*op| op.wrapInt(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + op.tracking(cg), + }), + else => |e| return e, + }; + }, + } + res[0] = ops[0].cmpInts(cmp_op, &ops[1], cg) catch |err| break :err err; + }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty.fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + if (opt_info) |*oi| { + for (ops) |op| for (res) |r| { + if (op.index == r.index) break; + } else try op.die(cg); + try cg.genCopy(.bool, oi.res[0].tracking(cg).short, res[0].tracking(cg).short, .{}); + try res[0].die(cg); + res[0] = oi.res[0]; + try cg.restoreState(oi.state, &oi.deaths, .{ + .emit_instructions = true, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + cg.performReloc(oi.reloc); + @memset(&ops, res[0]); + } + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .cmp_vector, .cmp_vector_optimized => |air_tag| fallback: { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const vector_cmp = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; + switch (vector_cmp.compareOperator()) { + .eq, .neq => {}, + .lt, .lte, .gte, .gt => if (cg.floatBits(cg.typeOf(vector_cmp.lhs).childType(zcu)) == null) + break :fallback try cg.airCmpVector(inst), + } + var ops = try cg.tempsFromOperands(inst, .{ vector_cmp.lhs, vector_cmp.rhs }); + var res: [1]Temp = undefined; + (err: switch (vector_cmp.compareOperator()) { + .lt, .lte, .gte, .gt => |cmp_op| { + switch (cmp_op) { + else => unreachable, + .lt, .lte => {}, + .gt, .gte => std.mem.swap(Temp, &ops[0], &ops[1]), + } + break :err cg.select(&res, &.{ty_pl.ty.toType()}, &ops, switch (@as(Condition, switch (cmp_op) { + else => unreachable, + .lt, .gt => .l, + .lte, .gte => .le, + })) { + else => unreachable, + inline .l, .le => |cc| comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .tmp0d, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .sse, .mem, .none } }, + .{ .src = .{ .mem, .sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) { + else => unreachable, + .l => .gt, + .le => .ge, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { + else => unreachable, + .l => .gt, + .le => .ge, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { + else => unreachable, + .l => .gt, + .le => .ge, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) { + else => unreachable, + .l => .gt, + .le => .ge, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { + else => unreachable, + .l => .gt, + .le => .ge, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { + else => unreachable, + .l => .gt, + .le => .ge, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, .p_w, .ackssd, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, -16), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ .@"1:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp3y, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .l => .lt, + .le => .le, + }), ._ }, + .{ ._, ._pd, .movmsk, .tmp4d, .tmp3x, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, + .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, + .{ ._, .fromCond(switch (cc) { + else => unreachable, + .l => .a, + .le => .ae, + }), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, + .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, + .{ ._, .fromCond(switch (cc) { + else => unreachable, + .l => .a, + .le => .ae, + }), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, + switch (cc) { + else => unreachable, + .l => .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_001), ._, ._ }, + .le => .{ ._, ._r, .sh, .tmp6h, .ui(1), ._, ._ }, + }, + .{ ._, .fromCond(switch (cc) { + else => unreachable, + .l => .z, + .le => .nc, + }), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, + .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, + .{ ._, .fromCond(switch (cc) { + else => unreachable, + .l => .a, + .le => .ae, + }), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp4t, .tmp5t, ._, ._ }, + .{ ._, .f_p, .st, .tmp5t, ._, ._, ._ }, + .{ ._, .fromCond(switch (cc) { + else => unreachable, + .l => .a, + .le => .ae, + }), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, + switch (cc) { + else => unreachable, + .l => .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_001), ._, ._ }, + .le => .{ ._, ._r, .sh, .tmp6h, .ui(1), ._, ._ }, + }, + .{ ._, .fromCond(switch (cc) { + else => unreachable, + .l => .z, + .le => .nc, + }), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + } }, + }); + }, + .eq, .neq => |cmp_op| cg.select(&res, &.{ty_pl.ty.toType()}, &ops, switch (@as(Condition, switch (cmp_op) { + else => unreachable, + .eq => .e, + .neq => .ne, + })) { + else => unreachable, + inline .e, .ne => |cc| comptime &.{ .{ + .src_constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm8, .none } }, + .{ .src = .{ .imm8, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .not, .dst0b, ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ }, + }, + } }, + }, .{ + .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .not, .dst0w, ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ }, + }, + } }, + }, .{ + .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .not, .dst0d, ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .mut_mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .not, .dst0q, ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ }, + }, + } }, + }, .{ + .src_constraints = .{ .any_bool_vec, .any_bool_vec, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .not, .tmp1p, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .byte, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .word, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .dword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .qword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_int = .{ .of = .xword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .byte, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_int = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .word, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_int = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .dword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .qword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_int = .{ .of = .qword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .byte, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .qword, .is = .word } }, + .{ .scalar_int = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .word, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .mmx, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .qword, .is = .dword } }, + .{ .scalar_int = .{ .of = .qword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .dword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .cmpeq, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_int = .{ .of = .yword, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .byte, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_int = .{ .of = .yword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .word, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_int = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .dword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_int = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ + .kind = .all, + .inverted = switch (cc) { + else => unreachable, + .e => false, + .ne => true, + }, + .scalar = .qword, + } } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, + .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp1), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp4x, ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ }, + .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp4y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, + .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b1111), ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"0:", ._dqu, .mov, .tmp4x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_q, .cmpeq, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b11), ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, + .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp4b, .si(0b1111), ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = switch (cc) { + else => unreachable, + .e => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + .ne => &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ ._, .p_, .xor, .tmp3q, .tmp3q, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp5q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .cmpeq, .tmp5q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .ackssd, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp5q, .tmp3q, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp5q, ._, ._ }, + .{ ._, ._, .xor, .tmp4b, .si(0b11), ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any_scalar_int, .any }, + .dst_constraints = .{ .{ .bool_vec = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, + .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any_scalar_int, .any }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, + .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any_scalar_int, .any }, + .dst_constraints = .{ .{ .bool_vec = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .sa(.src0p, .add_elem_limbs), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, + .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, + .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.tmp4, .add_size), ._, ._ }, + .{ ._, ._, .sub, .tmp2d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp2b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .sa(.dst0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .byte }, .{ .scalar_int_is = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, + .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .word }, .{ .scalar_int_is = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp4w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp4w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, + .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .scalar_int_is = .dword }, .{ .scalar_int_is = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp4d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, + .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_int_is = .qword }, .{ .scalar_int_is = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp4q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, + .{ ._, ._, .add, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .tmp0d, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .sse, .mem, .none } }, + .{ .src = .{ .mem, .sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .none } }, + .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .none } }, + }, + .dst_temps = .{ .{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp1x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp7d), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp8d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, .p_w, .ackssd, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, 16), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, -16), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ .@"1:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp3b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp1b), .tmp4b, ._, ._ }, + .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(64), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), .vp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp3y, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .memia(.src1x, .tmp0, .add_size), .sp(switch (cc) { + else => unreachable, + .e => .eq, + .ne => .neq, + }), ._ }, + .{ ._, ._pd, .movmsk, .tmp4d, .tmp3x, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6h, .si(0b1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_100), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6h, .si(0b1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp6h, .si(0b1_000_100), ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .dst_constraints = .{ .{ .bool_vec = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCond(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ }, + } }, + } }, + }), + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {s} {f} {f} {f}", .{ + @tagName(air_tag), + @tagName(vector_cmp.compareOperator()), + cg.typeOf(vector_cmp.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ vector_cmp.lhs, vector_cmp.rhs }, &ops, cg); + }, + .cond_br => try cg.airCondBr(inst), + .switch_br => try cg.airSwitchBr(inst), + .loop_switch_br => try cg.airLoopSwitchBr(inst), + .switch_dispatch => try cg.airSwitchDispatch(inst), + .@"try", .try_cold => try cg.airTry(inst), + .try_ptr, .try_ptr_cold => try cg.airTryPtr(inst), + .dbg_stmt => if (!cg.mod.strip) { + const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt; + _ = try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_line_stmt_line_column, + .data = .{ .line_column = .{ + .line = dbg_stmt.line, + .column = dbg_stmt.column, + } }, + }); + }, + .dbg_empty_stmt => if (!cg.mod.strip) { + if (cg.mir_instructions.len > 0) { + const prev_mir_op = &cg.mir_instructions.items(.ops)[cg.mir_instructions.len - 1]; + if (prev_mir_op.* == .pseudo_dbg_line_line_column) + prev_mir_op.* = .pseudo_dbg_line_stmt_line_column; + } + try cg.asmOpOnly(.{ ._, .nop }); + }, + .dbg_inline_block => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const dbg_inline_block = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + const old_inline_func = cg.inline_func; + defer cg.inline_func = old_inline_func; + cg.inline_func = dbg_inline_block.data.func; + if (!cg.mod.strip) _ = try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_enter_inline_func, + .data = .{ .ip_index = dbg_inline_block.data.func }, + }); + try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[dbg_inline_block.end..][0..dbg_inline_block.data.body_len])); + if (!cg.mod.strip) _ = try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_leave_inline_func, + .data = .{ .ip_index = old_inline_func }, + }); + }, + .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => |air_tag| if (!cg.mod.strip) { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + const air_name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); + const op_ty = cg.typeOf(pl_op.operand); + const local_ty = switch (air_tag) { + else => unreachable, + .dbg_var_ptr => op_ty.childType(zcu), + .dbg_var_val, .dbg_arg_inline => op_ty, + }; + var ops = try cg.tempsFromOperands(inst, .{pl_op.operand}); + var mcv = ops[0].tracking(cg).short; + switch (mcv) { + else => {}, + .eflags => |cc| switch (cc) { + else => {}, + // These values would self destruct. Maybe we make them use their + // turing complete dwarf expression interpreters for once? + .z_and_np, .nz_or_p => { + try cg.spillEflagsIfOccupied(); + mcv = ops[0].tracking(cg).short; + }, + }, + } + + try cg.mir_locals.append(cg.gpa, .{ + .name = switch (air_name) { + .none => switch (air_tag) { + else => unreachable, + .dbg_arg_inline => .none, + }, + else => try cg.addString(air_name.toSlice(cg.air)), + }, + .type = local_ty.toIntern(), + }); + + try cg.genLocalDebugInfo(air_tag, local_ty, ops[0].tracking(cg).short); + try ops[0].die(cg); + }, + .is_null => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const opt_ty = cg.typeOf(un_op); + const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + while (try ops[0].toBase(false, cg)) {} + try cg.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(cg).short.mem(cg, .{ + .size = if (!opt_repr_is_pl) + .byte + else if (opt_child_ty.isSlice(zcu)) + .ptr + else + .fromSize(opt_child_abi_size), + .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, + }), + .u(0), + ); + const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); + try is_null.finish(inst, &.{un_op}, &ops, cg); + }, + .is_non_null => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const opt_ty = cg.typeOf(un_op); + const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + while (try ops[0].toBase(false, cg)) {} + try cg.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(cg).short.mem(cg, .{ + .size = if (!opt_repr_is_pl) + .byte + else if (opt_child_ty.isSlice(zcu)) + .ptr + else + .fromSize(opt_child_abi_size), + .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, + }), + .u(0), + ); + const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); + try is_non_null.finish(inst, &.{un_op}, &ops, cg); + }, + .is_null_ptr => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const opt_ty = cg.typeOf(un_op).childType(zcu); + const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); + while (try ops[0].toLea(cg)) {} + try cg.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) + .byte + else if (opt_child_ty.isSlice(zcu)) + .ptr + else + .fromSize(opt_child_abi_size) }), + .u(0), + ); + const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); + try is_null.finish(inst, &.{un_op}, &ops, cg); + }, + .is_non_null_ptr => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const opt_ty = cg.typeOf(un_op).childType(zcu); + const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); + while (try ops[0].toLea(cg)) {} + try cg.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) + .byte + else if (opt_child_ty.isSlice(zcu)) + .ptr + else + .fromSize(opt_child_abi_size) }), + .u(0), + ); + const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); + try is_non_null.finish(inst, &.{un_op}, &ops, cg); + }, + .is_err => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const eu_ty = cg.typeOf(un_op); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + while (try ops[0].toBase(false, cg)) {} + try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ + .size = cg.memSize(eu_err_ty), + .disp = eu_err_off, + }), .u(0)); + const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); + try is_err.finish(inst, &.{un_op}, &ops, cg); + }, + .is_non_err => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const eu_ty = cg.typeOf(un_op); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + while (try ops[0].toBase(false, cg)) {} + try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ + .size = cg.memSize(eu_err_ty), + .disp = eu_err_off, + }), .u(0)); + const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); + try is_non_err.finish(inst, &.{un_op}, &ops, cg); + }, + .is_err_ptr => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const eu_ty = cg.typeOf(un_op).childType(zcu); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + try ops[0].toOffset(eu_err_off, cg); + while (try ops[0].toLea(cg)) {} + try cg.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), + .u(0), + ); + const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); + try is_err.finish(inst, &.{un_op}, &ops, cg); + }, + .is_non_err_ptr => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const eu_ty = cg.typeOf(un_op).childType(zcu); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + try cg.spillEflagsIfOccupied(); + var ops = try cg.tempsFromOperands(inst, .{un_op}); + try ops[0].toOffset(eu_err_off, cg); + while (try ops[0].toLea(cg)) {} + try cg.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), + .u(0), + ); + const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); + try is_non_err.finish(inst, &.{un_op}, &ops, cg); + }, + .load => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const val_ty = ty_op.ty.toType(); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{val_ty}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec_elem, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => res[0] = try ops[0].load(val_ty, .{ + .disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) { + .none => 0, + .runtime => unreachable, + else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), + }, + }, cg), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .ret => try cg.airRet(inst, false), + .ret_safe => try cg.airRet(inst, true), + .ret_load => try cg.airRetLoad(inst), + .store, .store_safe => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + cg.select(&.{}, &.{}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._r, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._s, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._s, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._s, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .leaa(.src0d, .add_vector_index_div_8_down_4), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .leaa(.src0d, .add_vector_index_div_8_down_4), .tmp0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._s, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => try ops[0].store(&ops[1], .{ + .disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) { + .none => 0, + .runtime => unreachable, + else => |vector_index| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)), + }, + .safe = switch (air_tag) { + else => unreachable, + .store => false, + .store_safe => true, + }, + }, cg), + else => |e| return e, + }; + for (ops) |op| try op.die(cg); + }, + .unreach => {}, + .fptrunc => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvtps2ph, .dst0q, .src0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvtps2ph, .dst0q, .src0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvtps2ph, .dst0x, .src0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncsfhf2" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp3d), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .mem(.tmp3d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncdfhf2" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp3d), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .mem(.tmp3d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsd2, .dst0x, .dst0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cvtsd2, .dst0x, .src0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsd2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtpd2, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .cvtpd2, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtpd2, .dst0x, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtpd2, .tmp1x, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .cvtpd2, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp0x), .dst0x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp4d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__truncxfhf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .mem(.tmp4d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp5w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0q, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp3d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp3d), .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .mem(.tmp3d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm1 }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .fpext => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfsf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_sd, .cvtss2, .dst0x, .dst0x, .dst0d, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_pd, .cvtps2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_pd, .cvtps2, .dst0y, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .zword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0y, .src0x, ._, ._ }, + .{ ._, .v_pd, .cvtps2, .tmp1y, .tmp0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_pd, .mova, .mem(.dst0y), .tmp1y, ._, ._ }, + .{ ._, .v_pd, .cvtps2, .tmp1y, .tmp0x, ._, ._ }, + .{ ._, .v_pd, .mova, .memd(.dst0y, 32), .tmp1y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_pd, .cvtps2, .tmp2y, .tmp1x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp1y, .ui(1), ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp2y, ._, ._ }, + .{ ._, .v_pd, .cvtps2, .tmp2y, .tmp1x, ._, ._ }, + .{ ._, .v_pd, .mova, .memsid(.dst0y, .@"4", .tmp0, 32), .tmp2y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfdf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f32, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_ss, .mov, .mem(.tmp1d), .tmp0x, ._, ._ }, + .{ ._, .f_, .ld, .mem(.tmp1d), ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm1 }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhfxf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp3d), .tmp2d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp3d), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendhftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtss2, .dst0x, .dst0x, .src0d, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cvtss2, .dst0x, .src0x, .src0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtss2, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0d, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cvtps2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .cvtps2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cvtps2, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .cvtps2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .cvtps2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0d, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0q, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0d, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0d, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendsftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extenddftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp0x), .dst0x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp0x), .dst0x, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__extendxftf2" } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .intcast => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const dst_ty = ty_op.ty.toType(); + const src_ty = cg.typeOf(ty_op.operand); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{dst_ty}, &ops, if (dst_ty.scalarType(zcu).abiSize(zcu) <= src_ty.scalarType(zcu).abiSize(zcu)) comptime &.{ .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0b, .mem(.src0b), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0w, .mem(.src0w), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .mem(.src0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0w, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .src0x, .dst0x, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b10_00_10_00), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b10_00_10_00), ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .yword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } } else comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0q, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0q, .src0b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .usize, .kind = .{ .reg = .edi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sd, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .eax } }, + .{ .type = .usize, .kind = .{ .reg = .edi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sd, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, + .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0q, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0q, .src0w, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .usize, .kind = .{ .reg = .edi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sd, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .eax } }, + .{ .type = .usize, .kind = .{ .reg = .edi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sd, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, + .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._d, .movsx, .dst0q, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._d, .movsx, .tmp0q, .src0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .usize, .kind = .{ .reg = .edi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sd, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0d, .ui(31), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .eax } }, + .{ .type = .usize, .kind = .{ .reg = .edi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sd, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_4), ._, ._ }, + .{ ._, .@"rep _sd", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._sq, .lod, ._, ._, ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa2(.dst0, .src0, .add_delta_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0w, .@"2", .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_d, .movsxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_d, .movzxb, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxb, .dst0x, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxb, .dst0x, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movsxb, .dst0x, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movzxb, .dst0x, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxb, .dst0y, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxb, .dst0y, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movsxb, .tmp1y, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"8", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movzxb, .tmp1y, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"8", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movsxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movzxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_q, .movsxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_q, .movzxb, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"8", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxb, .dst0x, .src0w, ._, ._ }, + .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxb, .dst0x, .src0w, ._, ._ }, + .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movsxb, .dst0x, .src0w, ._, ._ }, + .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movzxb, .dst0x, .src0w, ._, ._ }, + .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_d, .movsxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_d, .movzxw, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxw, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxw, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movsxw, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movzxw, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxw, .dst0y, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxw, .dst0y, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movsxw, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movzxw, .tmp1y, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"4", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movsxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movzxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_q, .movsxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_q, .movzxw, .tmp1x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"4", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1q, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxw, .dst0x, .src0d, ._, ._ }, + .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxw, .dst0x, .src0d, ._, ._ }, + .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movsxw, .dst0x, .src0d, ._, ._ }, + .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movzxw, .dst0x, .src0d, ._, ._ }, + .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklwd, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2q, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxd, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxd, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movsxd, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movzxd, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxd, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxd, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movsxd, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movzxd, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0y, .@"2", .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movsxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_q, .movzxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_q, .movsxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .p_q, .movzxd, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memsia(.dst0x, .@"2", .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._d, .movsx, .tmp1q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movsxd, .dst0x, .src0q, ._, ._ }, + .{ ._, .vp_q, .movsxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .movzxd, .dst0x, .src0q, ._, ._ }, + .{ ._, .vp_q, .movzxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movsxd, .dst0x, .src0q, ._, ._ }, + .{ ._, .p_q, .movsxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_q, .movzxd, .dst0x, .src0q, ._, ._ }, + .{ ._, .p_q, .movzxd, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpckldq, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .unpcklqdq, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._d, .movsx, .tmp2q, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._sq, .lod, ._, ._, ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .any_scalar_signed_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._sq, .lod, ._, ._, ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .any_scalar_int, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sa(.src0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa2(.dst0, .src0, .add_delta_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + dst_ty.fmt(pt), + src_ty.fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .intcast_safe => unreachable, + .trunc => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .exact_signed_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0d, .si(1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .exact_signed_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mem(.src0b), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .si(1), ._, ._ }, + .{ ._, ._, .neg, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .exact_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .mem(.src0b), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mem(.src0b), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0b, .sa(.dst0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_int, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mem(.src0b), ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .sa(.dst0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .exact_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .mem(.src0w), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mem(.src0w), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0w, .sa(.dst0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mem(.src0w), ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .sa(.dst0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .exact_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .any_int, .any, .any }, + .dst_constraints = .{ .{ .exact_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, + .{ ._, ._l, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0d, .sa(.dst0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .mem(.src0d), ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .sa(.dst0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .dst_constraints = .{ .{ .exact_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .mem(.src0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .mem(.src0q), ._, ._ }, + .{ ._, ._l, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .src0q, .dst0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .mem(.src0q), .dst0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .gpr }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .mem(.src0q), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .dst_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_dst0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_dst0_size, -16), ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_dst0_size, -8), ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .bzhi, .tmp2q, .memad(.src0q, .add_dst0_size, -16), .tmp2q, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp2q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .bzhi, .tmp2q, .memad(.src0q, .add_dst0_size, -8), .tmp2q, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .memad(.src0q, .add_dst0_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .memad(.src0q, .add_dst0_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 1 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_signed_int = .{ .of = .byte, .is = 1 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .si(1), ._, ._ }, + .{ ._, ._, .neg, .tmp1b, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsia(.src0b, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._l, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._l, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp2b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .@"and", .tmp2b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .@"and", .tmp2b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp2b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .sll, .dst0x, .src0x, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .sll, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .sll, .dst0y, .src0y, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .dst0y, .dst0y, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sll, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sll, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .sll, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_w, .sra, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp3w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .@"and", .tmp2w, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .sll, .dst0x, .src0x, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .sll, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .sll, .dst0y, .src0y, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .dst0y, .dst0y, .uia(32, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sll, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sll, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .sll, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_d, .sra, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"8", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._l, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp2d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp3d, .memi(.src0d, .tmp1), .tmp2d, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp3d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp1), ._, ._ }, + .{ ._, ._, .@"and", .tmp2d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_q, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), .tmp1q, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), .tmp1q, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"2", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), .tmp1q, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, + .{ ._, ._l, .sa, .tmp2q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp3q, .memi(.src0q, .tmp1), .tmp2q, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp2q, .memi(.src0q, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .leaa(.tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sa(.dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._l, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._l, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, + .{ ._, ._r, .sa, .tmp3q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._l, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._l, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .uia(64, .dst0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sa2(.src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp4q, .lea(.tmp1q), .tmp3q, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp4q, .lea(.tmp1q), .tmp3q, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp5q, .lea(.tmp1q), .tmp3q, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size_rem_64), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .bzhi, .tmp5q, .lea(.tmp1q), .tmp3q, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp3d, .sia(-2, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp3q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._, .mov, .lea(.tmp2q), .tmp3q, ._, ._ }, + .{ ._, ._, .mov, .lead(.tmp2q, 8), .si(0), ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(16, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .lead(.tmp2, 16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp4d, .sia(-1, .dst0, .add_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp4q, .lea(.tmp1q), ._, ._ }, + .{ ._, ._sq, .sto, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .sia2(8, .src0, .dst0, .add_delta_elem_size), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .optional_payload => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const pl = if (!hack_around_sema_opv_bugs or ty_op.ty.toType().hasRuntimeBitsIgnoreComptime(zcu)) + try ops[0].read(ty_op.ty.toType(), .{}, cg) + else + try cg.tempInit(ty_op.ty.toType(), .none); + try pl.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .optional_payload_ptr => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .optional_payload_ptr_set => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + if (!opt_ty.optionalReprIsPayload(zcu)) { + const opt_pl_ty = opt_ty.optionalChild(zcu); + const opt_pl_abi_size: i32 = @intCast(opt_pl_ty.abiSize(zcu)); + try ops[0].toOffset(opt_pl_abi_size, cg); + var has_value = try cg.tempInit(.bool, .{ .immediate = 1 }); + try ops[0].store(&has_value, .{}, cg); + try has_value.die(cg); + try ops[0].toOffset(-opt_pl_abi_size, cg); + } + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .wrap_optional => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const opt_ty = ty_op.ty.toType(); + const opt_pl_ty = cg.typeOf(ty_op.operand); + const opt_pl_abi_size: u31 = @intCast(opt_pl_ty.abiSize(zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var opt = try cg.tempAlloc(opt_ty); + try opt.write(&ops[0], .{}, cg); + if (!opt_ty.optionalReprIsPayload(zcu)) { + var has_value = try cg.tempInit(.bool, .{ .immediate = 1 }); + try opt.write(&has_value, .{ .disp = opt_pl_abi_size }, cg); + try has_value.die(cg); + } + try opt.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .unwrap_errunion_payload => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_pl_ty = ty_op.ty.toType(); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const pl = if (!hack_around_sema_opv_bugs or eu_pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) + try ops[0].read(eu_pl_ty, .{ .disp = eu_pl_off }, cg) + else + try cg.tempInit(eu_pl_ty, .none); + try pl.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .unwrap_errunion_err => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = cg.typeOf(ty_op.operand); + const eu_err_ty = ty_op.ty.toType(); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const err = try ops[0].read(eu_err_ty, .{ .disp = eu_err_off }, cg); + try err.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .unwrap_errunion_payload_ptr => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(eu_pl_off, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .unwrap_errunion_err_ptr => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); + const eu_err_ty = ty_op.ty.toType(); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(eu_err_off, cg); + const err = try ops[0].load(eu_err_ty, .{}, cg); + try err.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .errunion_payload_ptr_set => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(eu_err_off, cg); + var err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 }); + try ops[0].store(&err, .{}, cg); + try err.die(cg); + try ops[0].toOffset(eu_pl_off - eu_err_off, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .wrap_errunion_payload => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = ty_op.ty.toType(); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = cg.typeOf(ty_op.operand); + const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + const eu_pl_off: u31 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var eu = try cg.tempAlloc(eu_ty); + try eu.write(&ops[0], .{ .disp = eu_pl_off }, cg); + var err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 }); + try eu.write(&err, .{ .disp = eu_err_off }, cg); + try err.die(cg); + try eu.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .wrap_errunion_err => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = ty_op.ty.toType(); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var eu = try cg.tempAlloc(eu_ty); + try eu.write(&ops[0], .{ .disp = eu_err_off }, cg); + try eu.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .struct_field_ptr => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{struct_field.struct_operand}); + try ops[0].toOffset(@intCast(codegen.fieldOffset( + cg.typeOf(struct_field.struct_operand), + ty_pl.ty.toType(), + struct_field.field_index, + zcu, + )), cg); + try ops[0].finish(inst, &.{struct_field.struct_operand}, &ops, cg); + }, + .struct_field_ptr_index_0, + .struct_field_ptr_index_1, + .struct_field_ptr_index_2, + .struct_field_ptr_index_3, + => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(@intCast(codegen.fieldOffset( + cg.typeOf(ty_op.operand), + ty_op.ty.toType(), + switch (air_tag) { + else => unreachable, + .struct_field_ptr_index_0 => 0, + .struct_field_ptr_index_1 => 1, + .struct_field_ptr_index_2 => 2, + .struct_field_ptr_index_3 => 3, + }, + zcu, + )), cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .struct_field_val => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data; + const agg_ty = cg.typeOf(struct_field.struct_operand); + const field_ty = ty_pl.ty.toType(); + const field_off: u31 = switch (agg_ty.containerLayout(zcu)) { + .auto, .@"extern" => @intCast(agg_ty.structFieldOffset(struct_field.field_index, zcu)), + .@"packed" => unreachable, + }; + var ops = try cg.tempsFromOperands(inst, .{struct_field.struct_operand}); + var res = if (!hack_around_sema_opv_bugs or field_ty.hasRuntimeBitsIgnoreComptime(zcu)) + try ops[0].read(field_ty, .{ .disp = field_off }, cg) + else + try cg.tempInit(field_ty, .none); + try res.finish(inst, &.{struct_field.struct_operand}, &ops, cg); + }, + .set_union_tag => { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const union_ty = cg.typeOf(bin_op.lhs).childType(zcu); + const union_layout = union_ty.unionGetLayout(zcu); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + if (!hack_around_sema_opv_bugs or union_layout.tag_size > 0) try ops[0].store(&ops[1], .{ + .disp = @intCast(union_layout.tagOffset()), + }, cg); + const res = try cg.tempInit(.void, .none); + try res.finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .get_union_tag => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const union_ty = cg.typeOf(ty_op.operand); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + const union_layout = union_ty.unionGetLayout(zcu); + assert(union_layout.tag_size > 0); + const res = try ops[0].read(ty_op.ty.toType(), .{ + .disp = @intCast(union_layout.tagOffset()), + }, cg); + try res.finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .slice => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + try ops[0].toPair(&ops[1], cg); + try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .slice_len => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toSliceLen(cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .slice_ptr => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toSlicePtr(cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .ptr_slice_len_ptr => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(8, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .ptr_slice_ptr_ptr => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(0, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .array_elem_val => { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const array_ty = cg.typeOf(bin_op.lhs); + const res_ty = array_ty.elemType2(zcu); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm32, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .src0d, .ua(.none, .add_src1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .imm32, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .src0q, .ua(.none, .add_src1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .src0q, .src1q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .imm32, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .mema(.src0d, .add_src1_div_8_down_4), .ua(.none, .add_src1_rem_32), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .c }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .bt, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mema(.src0b, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .memi(.src0b, .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .mema(.src0w, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .memsi(.src0w, .@"2", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .mema(.src0d, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .memsi(.src0d, .@"4", .src1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .mema(.src0q, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .memsi(.src0q, .@"8", .src1), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => { + const elem_size = res_ty.abiSize(zcu); + var base = try cg.tempAllocReg(.usize, abi.RegisterClass.gp); + while (try ops[0].toBase(false, cg) or + try ops[1].toRegClass(true, .general_purpose, cg) or + try base.toRegClass(true, .general_purpose, cg)) + {} + const base_reg = base.tracking(cg).short.register.to64(); + const rhs_reg = ops[1].tracking(cg).short.register.to64(); + if (!std.math.isPowerOfTwo(elem_size)) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + rhs_reg, + rhs_reg, + .u(elem_size), + ); + try cg.asmRegisterMemory( + .{ ._, .lea }, + base_reg, + try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), + ); + } else if (elem_size > 8) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterImmediate( + .{ ._l, .sh }, + rhs_reg, + .u(std.math.log2_int(u64, elem_size)), + ); + try cg.asmRegisterMemory( + .{ ._, .lea }, + base_reg, + try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), + ); + } else try cg.asmRegisterMemory( + .{ ._, .lea }, + base_reg, + try ops[0].tracking(cg).short.mem(cg, .{ + .index = rhs_reg, + .scale = .fromFactor(@intCast(elem_size)), + }), + ); + // Hack around Sema insanity: lhs could be an arbitrarily large comptime-known array + // which could easily get spilled by the upcoming `load`, which would infinite recurse + // since spilling an array requires the same operation that triggered the spill. + try ops[0].die(cg); + ops[0] = base; + res[0] = try ops[0].load(res_ty, .{}, cg); + }, + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .slice_elem_val, .ptr_elem_val => { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + const res_ty = cg.typeOf(bin_op.lhs).elemType2(zcu); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + try ops[0].toSlicePtr(cg); + var res: [1]Temp = undefined; + if (!hack_around_sema_opv_bugs or res_ty.hasRuntimeBitsIgnoreComptime(zcu)) cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .leaa(.src0b, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .leai(.src0b, .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .leaa(.src0w, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .leasi(.src0w, .@"2", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .leaa(.src0d, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0d, .leasi(.src0d, .@"4", .src1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .leaa(.src0q, .add_src0_elem_size_mul_src1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .leasi(.src0q, .@"8", .src1), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => { + const elem_size = res_ty.abiSize(zcu); + while (true) for (&ops) |*op| { + if (try op.toRegClass(true, .general_purpose, cg)) break; + } else break; + const lhs_reg = ops[0].tracking(cg).short.register.to64(); + const rhs_reg = ops[1].tracking(cg).short.register.to64(); + if (!std.math.isPowerOfTwo(elem_size)) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + rhs_reg, + rhs_reg, + .u(elem_size), + ); + try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ .index = rhs_reg } }, + }); + } else if (elem_size > 8) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterImmediate( + .{ ._l, .sh }, + rhs_reg, + .u(std.math.log2_int(u64, elem_size)), + ); + try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ .index = rhs_reg } }, + }); + } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ + .index = rhs_reg, + .scale = .fromFactor(@intCast(elem_size)), + } }, + }); + res[0] = try ops[0].load(res_ty, .{}, cg); + }, + else => |e| return e, + } else res[0] = try cg.tempInit(res_ty, .none); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .slice_elem_ptr, .ptr_elem_ptr => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + try ops[0].toSlicePtr(cg); + const dst_ty = ty_pl.ty.toType(); + if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: { + const elem_size = dst_ty.childType(zcu).abiSize(zcu); + if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset; + while (true) for (&ops) |*op| { + if (try op.toRegClass(true, .general_purpose, cg)) break; + } else break; + const lhs_reg = ops[0].tracking(cg).short.register.to64(); + const rhs_reg = ops[1].tracking(cg).short.register.to64(); + if (!std.math.isPowerOfTwo(elem_size)) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + rhs_reg, + rhs_reg, + .u(elem_size), + ); + try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ .index = rhs_reg } }, + }); + } else if (elem_size > 8) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterImmediate( + .{ ._l, .sh }, + rhs_reg, + .u(std.math.log2_int(u64, elem_size)), + ); + try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ .index = rhs_reg } }, + }); + } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ + .index = rhs_reg, + .scale = .fromFactor(@intCast(elem_size)), + } }, + }); + } + try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .array_to_slice => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var len = try cg.tempInit(.usize, .{ + .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), + }); + try ops[0].toPair(&len, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .int_from_float, .int_from_float_optimized => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .dst0q, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .dst0d, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp1q, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .word }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttps2dq, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_, .cvttps2dq, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttps2dq, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_, .cvttps2dq, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_, .cvttps2dq, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .vp_w, .ackssd, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_, .cvttps2dq, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .vp_w, .ackusd, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_, .cvttps2dq, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp2q, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp2d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp2q, .tmp1x, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp2q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfdi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfdi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d0, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d0, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp1q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d0, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d0, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"8", .tmp0, 8), .tmp1q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .tmp2d, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", .v_ps, .xor, .tmp4x, .tmp4x, .tmp4x, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .vp_w, .insr, .tmp4x, .tmp4x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", .v_ps, .xor, .tmp4x, .tmp4x, .tmp4x, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .vp_w, .insr, .tmp4x, .tmp4x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .p_w, .insr, .tmp4x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .p_w, .insr, .tmp4x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixhfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp3d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp5x, .tmp4d, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunshfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp3d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp5x, .tmp4d, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttss2si, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttss2si, .dst0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttss2si, .dst0q, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttss2si, .dst0q, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_, .cvttss2si, .tmp2q, .src0d, ._, ._ }, + .{ ._, .v_ss, .sub, .tmp0x, .src0x, .lea(.tmp1d), ._ }, + .{ ._, .v_, .cvttss2si, .dst0q, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._, .cvttss2si, .tmp1q, .src0d, ._, ._ }, + .{ ._, ._ss, .sub, .src0x, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .cvttss2si, .dst0q, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src0x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, + .{ ._, .v_ps, .@"and", .src0x, .src0x, .lea(.tmp1x), ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_d, .mov, .tmp0d, .src0x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, + .{ ._, .v_ps, .@"and", .src0x, .src0x, .lea(.tmp1x), ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_q, .mov, .tmp1q, .src0x, ._, ._ }, + .{ ._, .vp_q, .extr, .tmp4q, .src0x, .ui(1), ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._d, .mov, .tmp0d, .src0x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, + .{ ._, ._ps, .@"and", .src0x, .lea(.tmp1x), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._d, .mov, .tmp0d, .src0x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, + .{ ._, ._ps, .@"and", .src0x, .lea(.tmp1x), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._q, .mov, .tmp1q, .src0x, ._, ._ }, + .{ ._, .p_q, .extr, .tmp4q, .src0x, .ui(1), ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._d, .mov, .tmp0d, .src0x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._d, .movsx, .tmp0q, .tmp0d, ._, ._ }, + .{ ._, ._ps, .@"and", .src0x, .lea(.tmp1x), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._q, .mov, .tmp1q, .src0x, ._, ._ }, + .{ ._, .p_d, .shuf, .src0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._q, .mov, .tmp4q, .src0x, ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .xor, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp4q, ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .vector_4_f32, .kind = .{ .smax_mem = .{} } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .tmp0x, .src0d, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .tmp0x, .lea(.tmp1x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._d, .movsx, .tmp4q, .src0d, ._, ._ }, + .{ ._, ._r, .sa, .tmp4q, .ui(63), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp4q, ._, ._ }, + .{ ._, ._, .xor, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .sub, .tmp1q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q, ._, ._ }, + .{ ._, ._, .sbb, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp5q, ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .lea, .tmp6p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .mem(.dst0x), .src0x, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .mem(.dst0x), .src0x, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .mov, .mem(.dst0q), .tmp1q0, ._, ._ }, + .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q1, ._, ._ }, + .{ ._, ._, .xor, .tmp1d0, .tmp1d0, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u32, .kind = .{ .reg = .ecx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .mem(.dst0x), .src0x, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memd(.dst0, 16), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-2, .dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .sto, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttps2dq, .tmp3x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttps2dq, .tmp3x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttss2si, .tmp1d, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttss2si, .tmp1d, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, .vp_w, .ackssd, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, .p_w, .ackssd, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, .vp_w, .ackusd, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, .p_w, .ackusd, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttps2dq, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttps2dq, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttps2dq, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_, .cvttss2si, .tmp1q, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttss2si, .tmp1q, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"4", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunssfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttsd2si, .dst0d, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttsd2si, .dst0d, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, + .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, + .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .fi_p, .st, .dst0w, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .fi_p, .st, .dst0d, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttsd2si, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttsd2si, .dst0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .fi_p, .st, .dst0q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .@"0x1p63_mem" = .{} } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_, .cvttsd2si, .tmp2q, .src0d, ._, ._ }, + .{ ._, .v_sd, .sub, .tmp0x, .src0x, .lea(.tmp1q), ._ }, + .{ ._, .v_, .cvttsd2si, .dst0q, .tmp0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp2q, ._, ._ }, + .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp2q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .@"0x1p63_mem" = .{} } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._, .cvttsd2si, .tmp1q, .src0d, ._, ._ }, + .{ ._, ._sd, .sub, .src0x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .cvttsd2si, .dst0q, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, + .{ .type = .i64, .kind = .mem }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .dst0b, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttpd2dq, .tmp3x, .memsi(.src0y, .@"8", .tmp0), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_d, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttpd2dq, .tmp3x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .dword, .to = .byte } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttpd2dq, .tmp3x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._d, .mov, .tmp4d, .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttsd2si, .tmp1d, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttsd2si, .tmp1d, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp1b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp2b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp2b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixsfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp0), .tmp3b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .ackssd, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, + .{ ._, .vp_w, .ackssd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .ackusd, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, + .{ ._, .vp_w, .ackusd, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttpd2dq, .tmp1x, .memsi(.src0y, .@"4", .tmp0), ._, ._ }, + .{ ._, .vp_w, .ackssd, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttpd2dq, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, .p_w, .ackssd, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._d, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttpd2dq, .tmp1x, .memsi(.src0y, .@"4", .tmp0), ._, ._ }, + .{ ._, .vp_w, .ackusd, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_q, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttpd2dq, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, .p_w, .ackusd, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._d, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttsd2si, .tmp1d, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .memi(.dst0w, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .i32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp2w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .cvttpd2dq, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .cvttpd2dq, .dst0x, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_, .cvttpd2dq, .tmp1x, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .cvttpd2dq, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._q, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .i64, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0q, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_, .cvttsd2si, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .cvttsd2si, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .memi(.dst0q, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memsid(.dst0q, .@"2", .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._sd, .mov, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsdfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp4x, .tmp4x, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .movl, .tmp4x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, + .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .dst0b, ._, ._ }, + .{ ._, .fi_p, .st, .tmp2w, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, .slow_incdec, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .movsx, .tmp4d, .tmp3b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .movsx, .tmp4d, .tmp3b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, + .{ ._, ._, .movsx, .tmp3d, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, + .{ ._, ._, .movsx, .tmp3d, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, .slow_incdec, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .tmp3b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .tmp3b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i16, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ .@"0:", .f_, .ld, .lea(.tmp0t), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp5w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fi_p, .stt, .dst0w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .fi_p, .st, .dst0w, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .memi(.dst0w, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .memi(.dst0w, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i32, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp2w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fi_p, .stt, .dst0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .fi_p, .st, .dst0d, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .memi(.dst0d, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .memi(.dst0d, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i64, .kind = .mem }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp2q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .i64, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fi_p, .stt, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, .fi_p, .st, .dst0q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .@"64bit", .sse3, .x87, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .i64, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp4q, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp4q, ._, ._ }, + .{ ._, .f_, .sub, .lea(.tmp0d), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, .fi_p, .stt, .tmp4q, ._, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, + .{ .type = .i64, .kind = .mem }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .dst0b, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_x87, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f32, .kind = .{ .@"0x1p63_mem" = .{} } }, + .{ .type = .i64, .kind = .mem }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_, .ld, .tmp0t, ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_11_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .dst0b, ._, ._ }, + .{ ._, ._, .lea, .dst0q, .mem(.tmp3), ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, .f_, .sub, .lea(.dst0d), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5q, .tmp4q, ._, ._ }, + .{ ._, .fi_p, .st, .tmp4q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp5q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .ui(63), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp4q, ._, ._ }, + .{ ._, ._, .@"or", .dst0q, .tmp5q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .x87, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .stt, .memi(.dst0q, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp0d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_11_00), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp0b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memsi(.src0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, .fi_p, .st, .memi(.dst0q, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp4q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .src0x, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .src0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .size = 16, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunsxfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } }, + .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } }, + .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } }, + .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .float = .xword }, .any, .any }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .int_from_float_safe => unreachable, + .int_from_float_optimized_safe => unreachable, + .float_from_int => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0q, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .dst0d, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + .{ .@"0:", .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .src0d, .src0b, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .src0d, .src0b, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .src0d, .src0w, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .src0d, .src0w, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memsi(.dst0x, .@"2", .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memsi(.dst0q, .@"2", .tmp0), .tmp1x, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memsi(.dst0x, .@"2", .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memsi(.dst0q, .@"2", .tmp0), .tmp1x, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memsi(.dst0w, .@"2", .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0w, .@"2", .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtdq2, .dst0x, .src0x, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtdq2, .dst0y, .src0y, ._, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .avx2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtdq2, .tmp1y, .memsi(.src0y, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtdq2, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memsi(.src0d, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .f16c, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_ss, .cvtsi2, .tmp1x, .tmp1x, .memsi(.src0q, .@"4", .tmp0), ._ }, + .{ ._, .v_, .cvtps2ph, .tmp1q, .tmp1x, .rm(.{}), ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse4_1, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse4_1, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4d, .tmp3d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp4d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d0, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d0, .mem(.tmp4d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp4d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d0, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp1d, .tmp3x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"8", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d0, .mem(.tmp4d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w0, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp4d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse4_1, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp2d, .tmp5x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp6d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse4_1, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp1), .tmp5x, .ui(0), ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp2d, .tmp5x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneihf" } }, + .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .mem }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .tmp6d, .tmp5d, ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp6d, ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp1), .tmp2w, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, + .{ ._, .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .dst0d, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", .v_ss, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .dst0x, .src0q, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .dst0d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._ss, .cvtsi2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateisf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneisf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ }, + .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0y, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ }, + .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0y, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ }, + .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ }, + .{ ._, ._ps, .cvtdq2, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .dst0y, .dst0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memsi(.dst0d, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtdq2, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .cvtdq2, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtdq2, .dst0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtdq2, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtdq2, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .cvtdq2, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .cvtsi2, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ps, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_ss, .cvtsi2, .tmp1x, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ss, .cvtsi2, .tmp1x, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"4", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneisf" } }, + .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .src0d, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i32, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1d, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0d, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i64, .kind = .mem }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .cvtsi2, .dst0x, .dst0x, .src0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .u32_0x1p52_hi_0x1p84_hi_0_0_mem }, + .{ .type = .vector_2_f64, .kind = .f64_0x1p52_0x1p84_mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_q, .mov, .tmp0x, .src0q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .unpckldq, .tmp0x, .tmp0x, .lea(.tmp1x), ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_pd, .sub, .tmp0x, .tmp0x, .lea(.tmp1x), ._ }, + .{ ._, .v_pd, .unpckh, .dst0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .u32_0x1p52_hi_0x1p84_hi_0_0_mem }, + .{ .type = .vector_2_f64, .kind = .f64_0x1p52_0x1p84_mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._q, .mov, .tmp0x, .src0q, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .unpckldq, .tmp0x, .lea(.tmp1x), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ }, + .{ ._, ._pd, .sub, .tmp0x, .lea(.tmp1x), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._pd, .unpckh, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + .{ ._, ._ns, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .src0q, .ui(1), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .@"or", .src0q, .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, .fi_, .ld, .dst0q, ._, ._, ._ }, + .{ ._, .f_, .add, .tmp1t, .tmp1t, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .dst0q, .src0q, ._, ._ }, + .{ ._, .fi_, .ld, .dst0q, ._, ._, ._ }, + .{ .@"0:", .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movsxb, .dst0x, .src0d, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movzxb, .dst0x, .src0d, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxb, .dst0x, .src0d, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"8", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, .vp_d, .movsxb, .tmp1x, .tmp1d, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, .p_d, .movsxb, .tmp1x, .tmp1d, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, ._pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"8", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, .vp_d, .movzxb, .tmp1x, .tmp1d, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, .p_d, .movzxb, .tmp1x, .tmp1d, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, ._pd, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movsxw, .dst0x, .src0q, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movsxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .movzxw, .dst0x, .src0q, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .dst0x, .dst0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .movzxw, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .dst0y, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .vp_d, .movsxw, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .p_d, .movsxw, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, ._pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .vp_d, .movzxw, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .p_d, .movzxw, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ }, + .{ ._, ._pd, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cvtdq2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .cvtdq2, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cvtdq2, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .cvtdq2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .cvtdq2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .cvtdq2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memsi(.dst0q, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_pd, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .v_sd, .cvtsi2, .tmp1x, .tmp1x, .memi(.src0q, .tmp0), ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._pd, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._sd, .cvtsi2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memsi(.src0q, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memsid(.src0q, .@"2", .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneidf" } }, + .{ .type = .f64, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .tmp0w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i32, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1d, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0d, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .{ .type = .i64, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, .fi_, .ld, .tmp1q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .signed_or_exclusive_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f32, .kind = .f32_0_0x1p64_mem }, + .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .memd(.src0d, 4), ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(31), ._, ._ }, + .{ ._, .f_, .add, .leasi(.tmp1d, .@"4", .tmp3), ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u1, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f32, .kind = .f32_0_0x1p64_mem }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fi_, .ld, .src0q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memd(.src0d, 4), ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(31), ._, ._ }, + .{ ._, .fn_cw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memd(.tmp2b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp2b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp2b, 1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .mem(.tmp4), ._, ._ }, + .{ ._, .f_, .add, .leasi(.tmp3d, .@"4", .tmp1), ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp2w, ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .mem }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3w, .tmp2w, ._, ._ }, + .{ ._, .fi_, .ld, .tmp3w, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .movsx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .movsx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .lea(.tmp1t), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ }, + .{ ._, ._, .movzx, .tmp3d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .fi_, .ld, .memi(.src0w, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"8", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"8", .tmp0), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .fi_, .ld, .memi(.src0d, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"4", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .fi_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatdixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memsi(.dst0t, .@"2", .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatundixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memsi(.dst0, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp1), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp1), ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneixf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .mov, .tmp3p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0b, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .src0w, ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memad(.dst0, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp2d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp4x, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.tmp1, -16), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"8", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunsitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatditf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatunditf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floattitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u128, .kind = .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q0, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp1q1, .memid(.src0q, .tmp0, 8), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuntitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floateitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__floatuneitf" } }, + .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp1), .tmp5x, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .leaa(.tmp0, .sub_src0_elem_size), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + cg.typeOf(ty_op.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .reduce => |air_tag| { + const nan = std.math.nan(f16); + const reduce = air_datas[@intFromEnum(inst)].reduce; + const res_ty = cg.typeOfIndex(inst); + var ops = try cg.tempsFromOperands(inst, .{reduce.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{res_ty}, &ops, switch (reduce.operation) { + .And, .Or, .Xor => switch (@as(Mir.Inst.Tag, switch (reduce.operation) { + else => unreachable, + .And => .@"and", + .Or => .@"or", + .Xor => .xor, + })) { + else => unreachable, + inline .@"and", .@"or", .xor => |mir_tag| comptime &switch (mir_tag) { + else => unreachable, + .@"and" => [_]Select.Case{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .@"test", .src0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_, .@"test", .src0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_b, .cmpeq, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"test", .src0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .@"test", .src0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .@"test", .src0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .cmpeq, .tmp0y, .tmp0y, .tmp0y, ._ }, + .{ ._, .vp_, .@"test", .src0y, .tmp0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.true) }, + .{ ._, .v_ps, .@"test", .src0y, .tmp0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .ptest_mask_mem = .src0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .all = 1 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"test", .src0y, .lea(.tmp0y), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .si(-1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .src0b, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .si(-1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .src0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .si(-1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .src0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .si(-1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .not, .src0q, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .src0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nc }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, + .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(-1), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(-8), ._, ._ }, + .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nc }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_2_smin), ._, ._ }, + .{ ._, ._, .@"or", .tmp0q, .mema(.src0q, .add_bit_size_div_8_down_8), ._, ._ }, + .{ ._, ._, .cmp, .tmp0q, .si(-1), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, + .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(-1), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(-8), ._, ._ }, + .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, + } }, + } }, + .@"or" => [_]Select.Case{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .@"test", .src0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_, .@"test", .src0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_b, .cmpeq, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"test", .src0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .ptest_mask_mem = .src0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"test", .src0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .@"test", .src0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .@"test", .src0y, .src0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .cmpeq, .tmp0y, .tmp0y, .tmp0y, ._ }, + .{ ._, .vp_, .@"test", .src0y, .tmp0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.true) }, + .{ ._, .v_ps, .@"test", .src0y, .tmp0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .any_bool_vec, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .ptest_mask_mem = .src0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ptest_cc = .{ .ref = .src0, .not = true, .all = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"test", .src0y, .lea(.tmp0y), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .ne }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .si(0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .ne }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .si(0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0w, .src0w, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .ne }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .si(0), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .ne }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0q, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .src0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nc }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, + .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"1b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nc }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"test", .mema(.src0q, .add_bit_size_div_8_down_8), .tmp0q, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, + .{ .@"1:", ._, .cmp, .memi(.src0q, .tmp0), .si(0), ._, ._ }, + .{ ._, ._ne, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"1b", ._, ._, ._ }, + } }, + } }, + .xor => [_]Select.Case{ .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0b, .src0b, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .popcnt, .tmp0w, .src0w, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, .fast_imm16, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .popcnt, .src0w, .src0w, ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .popcnt, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ua(.src0b, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, .false_deps_popcnt, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .popcnt, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, .false_deps_popcnt, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(16), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .popcnt, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .src0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .popcnt, .src0d, .src0d, ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(16), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ua(.src0w, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(16), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ua(.src0w, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .popcnt, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, .false_deps_popcnt, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .src0q, .src0q, ._, ._ }, + .{ ._, ._, .@"and", .src0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .popcnt, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_bool_vec = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .ui(32), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .src0q, ._, ._ }, + .{ ._, ._, .popcnt, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .ui(32), ._, ._ }, + .{ ._, ._, .@"and", .tmp0d, .ua(.src0d, .add_umax), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .src0d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp1q, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .popcnt, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .exact_remainder_bool_vec = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_bit_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .tmp1q, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1q, .ui(32), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .popcnt, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .mema(.src0q, .add_bit_size_div_8_down_8), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp0q, .memi(.src0q, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .popcnt, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .@"and", .tmp0b, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .bool, .any }, + .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .po }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-8, .src0, .add_bit_size_div_8_down_8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .mema(.src0q, .add_bit_size_div_8_down_8), ._, ._ }, + .{ .@"0:", ._, .xor, .tmp0q, .memi(.src0q, .tmp1), ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0q, .ui(32), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(16), ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp1d, .ui(8), ._, ._ }, + .{ ._, ._, .xor, .tmp0b, .tmp1b, ._, ._ }, + } }, + } }, + } ++ [_]Select.Case{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, mir_tag, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, mir_tag, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, mir_tag, .dst0w, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, mir_tag, .dst0d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_, mir_tag, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, mir_tag, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .tmp0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .tmp0x, .memd(.src0x, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .tmp0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_2_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_1_u128, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, mir_tag, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_, mir_tag, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .yword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .yword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .yword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .yword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .{ .type = .vector_1_u256, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_, mir_tag, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, mir_tag, .dst0y, .dst0y, .tmp2y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .yword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .yword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .yword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ + .ref = .src0, + .invert = switch (mir_tag) { + else => unreachable, + .@"and" => true, + .@"or", .xor => false, + }, + } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, switch (mir_tag) { + else => unreachable, + .@"and" => .@"or", + .@"or", .xor => .@"and", + }, .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .v_ps, mir_tag, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memi(.src0, .tmp0), ._, ._ }, + .{ .@"1:", ._, .mov, .tmp3q, .leai(.tmp2q, .tmp1), ._, ._ }, + .{ ._, ._, mir_tag, .memi(.dst0q, .tmp1), .tmp3q, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }, + .Min => comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_b, .mins, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_b, .minu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_b, .mins, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_b, .minu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_b, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_b, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_b, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_b, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_b, .mins, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_b, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_b, .mins, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_b, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .vp_b, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_b, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, .p_b, .mins, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_b, .minu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_b, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_b, .minu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_b, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_b, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .minu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mins, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .minu, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .minu, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_w, .sub, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_w, .minposu, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .ph_w, .minposu, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_ps, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp0x, .lea(.tmp1d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_w, .mins, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp1x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_w, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_w, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_w, .mins, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_w, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .mins, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_w, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .vp_w, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._, .movddup, .tmp4x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_w, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mins, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_w, .minu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_w, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_w, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_d, .mins, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_d, .minu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_d, .mins, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_d, .minu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .mins, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_d, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .mins, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .mins, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .vp_d, .mins, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .mins, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_d, .mins, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .mins, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_d, .minu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_d, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .minu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_d, .minu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_d, .minu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .minu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .minu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .minu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nl, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nb, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .v_i128, .extract, .dst0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .v_f128, .extract, .dst0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp1y, .tmp2y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .memad(.src0y, .add_unaligned_size, -32), ._ }, + .{ .@"0:", .vp_, .xor, .tmp3y, .tmp2y, .memi(.src0y, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .memad(.src0x, .add_unaligned_size, -16), ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp2x, .memi(.src0x, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smax } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memad(.src0y, .add_size, -32), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -64), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ .@"0:", .vp_, .xor, .tmp4x, .tmp2x, .memi(.src0x, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nl, .cmov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nb, .cmov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._nl, .cmov, .tmp1p, .tmp3p, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._l, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._nb, .cmov, .tmp1p, .tmp3p, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp2y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp0y, .tmp0y, .tmp2y, .tmp1y }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, .v_ps, .min, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .min, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .@"or", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .min, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .min, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ .@"0:", .v_ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .min, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._sd, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._sd, .min, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .min, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .min, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_sd, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_pd, .min, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_pd, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_pd, .min, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_sd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._pd, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._sd, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._pd, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._pd, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._sd, .min, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_pd, .min, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ .@"0:", .v_pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_pd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._pd, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._sd, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = nan } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._pd, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._sd, .min, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + .Max => comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_b, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_b, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_b, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_b, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_64_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_b, .maxs, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_b, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_b, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .vp_b, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_b, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxs, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i8, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp3x, .ui(8), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_b, .maxu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_b, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_b, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_b, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .maxu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .signed_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .unsigned_int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .tmp1b, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .maxs, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .maxu, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxu, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .add, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .@"or", .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp3x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .cmpeq, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2y, .lea(.tmp0d), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_ps, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp0x, .lea(.tmp1d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp2y, .lea(.tmp0w), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_w, .minu, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .minu, .dst0x, .dst0x, .tmp4x, ._ }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp0x, .lea(.tmp1w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp1x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_w, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp2x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ .@"0:", .vp_w, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp1x, .tmp1x, ._, ._ }, + .{ .@"0:", .p_w, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_w, .maxs, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_w, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp3x, .lea(.tmp0w), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .vp_w, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp3x, .lea(.tmp0d), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smax, .outside = .smax } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._, .movddup, .tmp4x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i16, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_w, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .maxs, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_w, .maxu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_w, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_w, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .cmpeq, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_w, .cmpeq, .tmp2x, .tmp2x, ._, ._ }, + .{ .@"0:", .p_w, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .ph_w, .minposu, .dst0x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_w, .subus, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .subus, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .signed_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movsx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .unsigned_int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .maxs, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .maxs, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_d, .maxs, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .vp_d, .maxs, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxs, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .p_d, .maxs, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxs, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp4x, ._, ._ }, + .{ ._, .p_, .andn, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp4x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_d, .maxu, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .maxu, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_d, .maxu, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_d, .maxu, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .maxu, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .maxu, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .maxu, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .tmp5x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .signed_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .unsigned_int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .tmp1d, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .tmp0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp0x, .dst0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .v_i128, .extract, .dst0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .v_f128, .extract, .dst0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .tmp3x, .dst0x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .tmp0x, .tmp1x }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .v_pd, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"and", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1y, .tmp0y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp0y, .tmp1y }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp0x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .mem(.src0y), ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2y, .tmp1y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp1y, .tmp2y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp2x, .tmp1x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp1x, .tmp2x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp1x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .memad(.src0y, .add_unaligned_size, -32), ._ }, + .{ .@"0:", .vp_, .xor, .tmp3y, .tmp2y, .memi(.src0y, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .memad(.src0x, .add_unaligned_size, -16), ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp2x, .memi(.src0x, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp3y, .tmp4y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp4x, .tmp3x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp3x, .tmp4x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp3x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp4x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp3x, .tmp4x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memad(.src0y, .add_size, -32), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -64), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"and", .tmp4y, .tmp4y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp4y, .tmp4y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp4y, .tmp2y, .memi(.src0y, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp4y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .dst0y, .tmp4y, .tmp5y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_size, -32), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .xor, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ .@"0:", .vp_, .xor, .tmp4x, .tmp2x, .memi(.src0x, .tmp0), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp4x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .dst0x, .tmp4x, .tmp5x }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .splat_int_mem = .{ .ref = .src0, .inside = .smin, .outside = .smin } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp4x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_, .xor, .tmp4x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .dst0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .tmp4x, .tmp5x, ._ }, + .{ ._, .p_, .xor, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._l, .cmov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .signed_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._b, .cmov, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .unsigned_int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .tmp1q, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .tmp1q, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1p, .tmp3p, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sia(1, .dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasiad(.tmp1q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasiad(.tmp3q, .@"8", .tmp2, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .leaad(.tmp1q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leaad(.tmp3q, .add_src0_elem_size, -8), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._b, .cmov, .tmp1p, .tmp3p, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp2p, .sa(.dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp4q, .leasia(.tmp1q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp4q, .leasia(.tmp3q, .@"8", .tmp2, .add_src0_elem_size), ._, ._ }, + .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .tmp3p, ._, ._ }, + .{ .@"1:", ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp2y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp0y, .tmp0y, .tmp2y, .tmp1y }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, .v_ps, .max, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp5y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp3y, .tmp3y, .tmp5y, .tmp4y }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp5y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .cmp, .tmp4y, .tmp4y, .tmp4y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp5y, .tmp4y }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .max, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .blendv, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .@"and", .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .@"or", .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .max, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .max, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_ps, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ss, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ss, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ .@"0:", .v_ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ss, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_ps, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._ps, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ps, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._ps, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .max, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._sd, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp1x, .src0x, ._, ._ }, + .{ ._, ._sd, .max, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp0x, .tmp0x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp0x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .max, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .tmp0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .max, .tmp1x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_sd, .cmp, .dst0x, .dst0x, .dst0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .tmp0x, .tmp1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .tmp1x, .src0x, .tmp0x, ._ }, + .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .src0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_pd, .max, .tmp1y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_pd, .cmp, .tmp0y, .tmp0y, .tmp0y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp1y, .tmp0y }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_pd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .tmp1x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_sd, .cmp, .tmp0x, .tmp0x, .tmp0x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp1x, .tmp0x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_pd, .max, .tmp2y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp2y, .tmp1y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_pd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .tmp2x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_sd, .cmp, .tmp1x, .tmp1x, .tmp1x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp2x, .tmp1x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._pd, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._sd, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp2x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._pd, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._pd, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._sd, .max, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp1x, .tmp1x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp3y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_pd, .max, .tmp4y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .tmp3y, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0y, .dst0y, .tmp4y, .tmp3y }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ .@"0:", .v_pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_pd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_pd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .tmp4x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_sd, .cmp, .tmp3x, .tmp3x, .tmp3x, .vp(.ord) }, + .{ ._, .v_pd, .blendv, .dst0x, .dst0x, .tmp4x, .tmp3x }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_2_f64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._pd, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._sd, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.ord), ._ }, + .{ ._, ._pd, .blendv, .dst0x, .tmp4x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -nan } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._pd, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._pd, .mova, .tmp4x, .dst0x, ._, ._ }, + .{ ._, ._sd, .max, .tmp4x, .tmp3x, ._, ._ }, + .{ ._, ._sd, .cmp, .tmp3x, .tmp3x, .sp(.unord), ._ }, + .{ ._, ._pd, .@"and", .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._pd, .andn, .tmp3x, .tmp4x, ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp2t, ._, ._ }, + .{ ._, .f_u, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ ._, .f_, .xam, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b1_000_100), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .xch, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + .Add => comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp0x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_b, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .tmp0x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_b, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_b, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_b, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_b, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_b, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_b, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_b, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_b, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_b, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .add, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .add, .dst0b, .memi(.src0b, .tmp0), ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_w, .add, .dst0x, .src0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_w, .add, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_w, .add, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_w, .add, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .ph_w, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_w, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_w, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_w, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_w, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_w, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_w, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_w, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", ._, .add, .dst0w, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_d, .add, .dst0x, .src0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .ph_d, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_d, .add, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .ph_d, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vph_d, .add, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .ph_d, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_d, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_d, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_d, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_d, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vph_d, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_d, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_d, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .ph_d, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_d, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", ._, .add, .dst0d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_q, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_q, .add, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .lead(.tmp0y, 32), ._ }, + .{ ._, .vp_q, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_q, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_q, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_q, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_q, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .vp_q, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_q, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_q, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ .@"0:", .vp_q, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_q, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", .p_q, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_q, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", ._, .add, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1p, .sa(.dst0, .sub_size_div_8), ._, ._ }, + .{ ._, ._, .lea, .tmp2q, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp3q, .leasia(.tmp2q, .@"8", .tmp1, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .memsia(.dst0q, .@"8", .tmp1, .add_size), .tmp3q, ._, ._ }, + .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .vh_ps, .add, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, + .{ .@"0:", .vp_w, .insr, .dst0x, .dst0x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(1) }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, + .{ .@"0:", .vp_w, .insr, .dst0x, .dst0x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(1) }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .mem(.src0w), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .mem(.src0w), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_ps, .add, .dst0x, .src0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_ps, .add, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .add, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .v_ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, + .{ .@"0:", .v_ss, .add, .dst0x, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, + .{ .@"0:", ._ss, .add, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_pd, .add, .dst0x, .src0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .h_pd, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_pd, .add, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .v_sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, + .{ .@"0:", .v_sd, .add, .dst0x, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, + .{ .@"0:", ._sd, .add, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, + .{ .@"0:", .f_, .add, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + .Mul => comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .word, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .srl, .dst0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .dst0x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .tmp0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_q, .mov, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_b, .sub, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._q, .mov, .tmp2x, .lea(.tmp0q), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0x, .src0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp0x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_d, .shuf, .dst0x, .tmp0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_b, .sub, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp2x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .srl, .tmp0y, .src0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .src0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .src0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp1x, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .tmp0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .tmp0x, .tmp0x, .tmp1x, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_b, .sub, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_w, .srl, .tmp2y, .dst0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_b, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_b, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .tmp2x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .vp_w, .srl, .tmp1y, .dst0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, .vp_w, .srl, .tmp1y, .tmp0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .tmp0y, .tmp0y, .tmp1y, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_b, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_b, .sub, .dst0y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_w, .srl, .tmp3y, .tmp2y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_w, .srl, .tmp3y, .dst0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ ._, .vp_w, .srl, .tmp1y, .dst0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .vp_w, .srl, .tmp1x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ .@"0:", .p_w, .movzxb, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp1x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-96, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_b, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_b, .sub, .dst0y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_w, .srl, .tmp3y, .tmp2y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_w, .srl, .tmp3y, .dst0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memid(.src0x, .tmp0, 16), ._, ._ }, + .{ ._, .vp_w, .mull, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .vp_w, .movzxb, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_b, .sub, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_w, .srl, .tmp2y, .dst0y, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-40, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_b, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_b, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .dst0x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .vp_w, .srl, .tmp3x, .tmp2x, .ui(8), ._ }, + .{ ._, .vp_w, .mull, .tmp2x, .tmp2x, .tmp3x, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_w, .movzxb, .tmp2x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", .p_w, .movzxb, .tmp2x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-24, .src0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_b, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_w, .srl, .tmp2x, .ui(8), ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp2x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .al }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .mul, .memi(.src0b, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .byte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .al }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0b, .add_unaligned_size, -1), ._, ._ }, + .{ .@"0:", ._, .mul, .memi(.src0b, .tmp0), ._, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, .vp_, .andn, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_w, .sub, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_w, .sub, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_w, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_w, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .tmp0x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_w, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_w, .sub, .dst0y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_w, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_w, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp1x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_w, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_w, .sub, .dst0y, .tmp3y, .dst0y, ._ }, + .{ .@"0:", .vp_w, .mull, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_w, .sub, .dst0y, .dst0y, .tmp2y, ._ }, + .{ .@"0:", .vp_w, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_w, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_w, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_w, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", .p_w, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .shufl, .tmp2x, .dst0x, .ui(0b01), ._ }, + .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .dst0d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ .@"0:", .i_, .mul, .dst0w, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .tmp0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, .vp_, .andn, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_d, .sub, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_d, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_d, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_dqa, .mov, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .vp_d, .mull, .dst0y, .dst0y, .memd(.src0y, 32), ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .vp_d, .shuf, .tmp0x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .vp_d, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .vp_d, .sub, .dst0y, .tmp3y, .dst0y, ._ }, + .{ ._, .vp_d, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .vp_d, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .p_d, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .tmp1x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_d, .sub, .tmp2y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .andn, .tmp3y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .vp_d, .sub, .dst0y, .tmp3y, .dst0y, ._ }, + .{ .@"0:", .vp_d, .mull, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .mull, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_size), ._, ._ }, + .{ ._, .vp_, .andn, .dst0y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_d, .sub, .dst0y, .dst0y, .tmp2y, ._ }, + .{ .@"0:", .vp_d, .mull, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .andn, .tmp3y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .vp_d, .sub, .dst0x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp3y, .ui(1), ._ }, + .{ ._, .vp_d, .sub, .tmp2x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_d, .mull, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .vp_d, .mull, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .vp_udq, .mul, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_d, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", .p_d, .mull, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0, .invert = true } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .p_d, .sub, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .tmp2x, .ui(0b11_11_01_01), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .p_d, .shuf, .tmp2x, .dst0x, .ui(0b11_10), ._ }, + .{ ._, .p_udq, .mul, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .int = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0d, .memad(.src0d, .add_unaligned_size, -4), ._, ._ }, + .{ .@"0:", .i_, .mul, .dst0d, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .int = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .dst0q, .memad(.src0q, .add_unaligned_size, -8), ._, ._ }, + .{ .@"0:", .i_, .mul, .dst0q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .adx, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .isize, .kind = .{ .reg = .rcx } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"1:", ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, ._, .@"or", .tmp4q, .leai(.tmp1q, .tmp3), ._, ._ }, + .{ ._, ._z, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .leaad(.tmp3, .sub_src0_elem_size, 8), ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ .@"2:", ._x, .mul, .tmp8q, .tmp7q, .leai(.tmp2q, .tmp5), ._ }, + .{ ._, ._x, .adc, .tmp7q, .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp5, .add_src0_elem_size, -8), .tmp7q, ._, ._ }, + .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ }, + .{ ._, ._x, .ado, .tmp8q, .memia(.dst0q, .tmp5, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp6q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp5p, .lead(.tmp5, 8), ._, ._ }, + .{ ._, ._mp, .j, .@"2b", ._, ._, ._ }, + .{ .@"3:", ._, .mov, .memi(.dst0q, .tmp3), .tmp4q, ._, ._ }, + .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp3d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, + .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, + .{ .@"3:", ._x, .mul, .tmp9q, .tmp8q, .leasi(.tmp2q, .@"8", .tmp4), ._ }, + .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, + .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, + .{ .@"3:", ._x, .mul, .tmp9q, .tmp8q, .leasi(.tmp2q, .@"8", .tmp4), ._ }, + .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, + .{ .@"3:", ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp4), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._, .sub, .tmp3d, .si(1), ._, ._ }, + .{ ._, ._ae, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(8, .src0, .add_unaligned_size_sub_2_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mema(.src0, .add_unaligned_size_sub_elem_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sia(-1, .dst0, .add_size_div_8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp4p, .leaa(.tmp3, .sub_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"or", .tmp5q, .leasi(.tmp1q, .@"8", .tmp3), ._, ._ }, + .{ ._, ._nz, .j, .@"3f", ._, ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp3), .tmp5q, ._, ._ }, + .{ ._, ._mp, .j, .@"4f", ._, ._, ._ }, + .{ .@"2:", ._, .adc, .tmp9q, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), ._, ._ }, + .{ ._, ._, .adc, .tmp6b, .si(0), ._, ._ }, + .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ }, + .{ .@"3:", ._, .mov, .tmp8q, .tmp5q, ._, ._ }, + .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp4), ._, ._, ._ }, + .{ ._, ._l, .sh, .tmp6b, .ui(4), ._, ._ }, + .{ ._, ._, .adc, .tmp8q, .tmp7q, ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp4, .add_src0_elem_size), .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp4p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"2b", ._, ._, ._ }, + .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp3d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.dst0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, + .{ .@"0:", .vp_w, .insr, .dst0x, .dst0x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(1) }, + .{ ._, .v_ps, .cvtph2, .dst0x, .dst0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .mem(.src0w), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .mem(.src0w), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .mem(.src0w), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .mul, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .mul, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .v_ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, + .{ .@"0:", .v_ss, .mul, .dst0x, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(4, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.src0d), ._, ._ }, + .{ .@"0:", ._ss, .mul, .dst0x, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._sd, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .mul, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .v_sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, + .{ .@"0:", .v_sd, .mul, .dst0x, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._sd, .mov, .dst0x, .mem(.src0q), ._, ._ }, + .{ .@"0:", ._sd, .mul, .dst0x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .dst0q, ._, ._, ._ }, + .{ .@"1:", .f_, .mul, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .mem(.src0x), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .mem(.src0x), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s}.{s} {f} {f}", .{ + @tagName(air_tag), + @tagName(reduce.operation), + cg.typeOf(reduce.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + switch (reduce.operation) { + .And, .Or, .Xor, .Min, .Max => {}, + .Add, .Mul => if (cg.intInfo(res_ty)) |_| res[0].wrapInt(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s}.{s} wrap {f} {f}", .{ + @tagName(air_tag), + @tagName(reduce.operation), + res_ty.fmt(pt), + res[0].tracking(cg), + }), + else => |e| return e, + }, + } + try res[0].finish(inst, &.{reduce.operand}, &ops, cg); + }, + .reduce_optimized => |air_tag| { + const inf = std.math.inf(f16); + const reduce = air_datas[@intFromEnum(inst)].reduce; + var ops = try cg.tempsFromOperands(inst, .{reduce.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (reduce.operation) { + .And, .Or, .Xor => unreachable, + .Min => comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp0y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fminh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .min, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .min, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .min, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .min, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .min, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .min, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ss, .min, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .min, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .v_ps, .min, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .min, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ss, .min, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._sd, .min, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .min, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_pd, .min, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._sd, .min, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .min, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_pd, .min, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_pd, .min, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .min, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .v_pd, .min, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .min, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = inf } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .min, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._sd, .min, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_nb, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._z, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fminq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + .Max => comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp0y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmaxh" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .max, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .max, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .max, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .max, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .max, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .max, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ss, .max, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .max, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .v_ps, .max, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .max, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ss, .max, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._sd, .max, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .max, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_pd, .max, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._sd, .max, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .max, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_pd, .max, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_pd, .max, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .max, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .v_pd, .max, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .max, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = -inf } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .max, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._sd, .max, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_b, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .{ .type = .f64, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucomi, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_b, .cmov, .tmp2t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + .{ ._, ._b, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp3h, .si(0b0_000_001), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, .f_, .st, .tmp1t, ._, ._, ._ }, + .{ .@"1:", .f_p, .st, .tmp2t, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + .Add => comptime &.{ .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .vh_ps, .add, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .add, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .add, .tmp0y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .tmp2y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .add, .tmp2y, .tmp2y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .tmp2x, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_ps, .add, .dst0x, .src0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_ps, .add, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .add, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_ps, .add, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .vh_ps, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .add, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .v_ps, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .v_ps, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, .vh_ps, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .v_ps, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .shuf, .tmp2x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, + .{ ._, .h_ps, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .add, .dst0x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp2x, .tmp2x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_pd, .add, .dst0x, .src0x, .src0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .h_pd, .add, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._sd, .add, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_pd, .add, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .add, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vh_pd, .add, .dst0y, .src0y, .src0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memd(.src0y, 32), ._ }, + .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_i128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vh_pd, .add, .dst0y, .dst0y, .dst0y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._sd, .add, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ ._, .v_pd, .@"and", .tmp2y, .tmp2y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ .@"0:", .v_pd, .add, .tmp2y, .tmp2y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_pd, .add, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_pd, .add, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .vh_pd, .add, .dst0x, .dst0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_f128, .extract, .tmp2x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + .{ .@"0:", .v_pd, .add, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp2x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .add, .dst0x, .dst0x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, .fast_hops, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .h_pd, .add, .dst0x, .dst0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .add, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp2x, .dst0x, ._, ._ }, + .{ ._, ._sd, .add, .dst0x, .tmp2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .add, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp1), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .add, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + .Mul => comptime &.{ .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .dword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .mul, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .tmp0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .tmp0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .mul, .tmp0y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .tmp1x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_f128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp1y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_32_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-80, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .tmp3y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .mul, .tmp3y, .tmp3y, .tmp4y, ._ }, + .{ ._, .v_f128, .extract, .tmp4x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp4y, .tmp4x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp4y, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f16, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .cvtph2, .dst0y, .dst0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .tmp3x, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp3y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .dst0x, .dst0x, .dst0x, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .insr, .dst0x, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0) }, + .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .insr, .dst0x, .memad(.src0w, .add_unaligned_size, -2), .ui(0), ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .word }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .dst0x, .dst0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memad(.src0w, .add_unaligned_size, -2), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .dst0x, .mem(.tmp2d), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .qword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ss, .mul, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .mul, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .mul, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .tmp0x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_ps, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mul, .dst0x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp1x, .tmp1x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_16_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_ps, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .mul, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_ps, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .v_ps, .mul, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .shuf, .tmp3x, .dst0x, .dst0x, .ui(0b01) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .mul, .dst0x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp3x, .tmp3x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._sd, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_sd, .mul, .tmp0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .tmp0x, .dst0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_pd, .mul, .dst0x, .src0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .mem(.src0y), ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memd(.src0y, 32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_i128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .exact_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .mova, .dst0y, .mem(.src0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp0y, .memd(.src0y, 32), ._, ._ }, + .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp0y, ._ }, + .{ ._, .v_i128, .extract, .tmp0x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-64, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .memad(.src0y, .add_unaligned_size, -32), ._, ._ }, + .{ .@"0:", .v_pd, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_f128, .extract, .tmp1x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + .{ ._, .v_ps, .movhl, .tmp1x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .unaligned_multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._pd, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp1x, .dst0x, ._, ._ }, + .{ ._, ._sd, .mul, .dst0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx512f, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_64_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_8_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .lead(.tmp0y, 32), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .tmp3y, .tmp3y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -64), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .v_pd, .@"or", .tmp3y, .tmp3y, .lead(.tmp0y, 32), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-128, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .mul, .tmp3y, .tmp3y, .memid(.src0y, .tmp0, 32), ._ }, + .{ ._, .v_pd, .mul, .dst0y, .dst0y, .memi(.src0y, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(64), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_pd, .mul, .dst0y, .dst0y, .tmp3y, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_4_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .dst0y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0y, .dst0y, .memad(.src0y, .add_size, -32), ._ }, + .{ ._, .v_pd, .@"or", .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-48, .src0, .add_size), ._, ._ }, + .{ ._, .v_f128, .extract, .tmp3x, .dst0y, .ui(1), ._ }, + .{ ._, .v_pd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + .{ .@"0:", .v_pd, .mul, .dst0x, .dst0x, .memi(.src0x, .tmp0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_ps, .movhl, .tmp3x, .dst0x, .dst0x, ._ }, + .{ ._, .v_sd, .mul, .dst0x, .dst0x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pand_mask_mem = .{ .ref = .src0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .splat_float_mem = .{ .ref = .src0, .outside = 1.0 } } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, ._pd, .@"and", .dst0x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, ._pd, .@"or", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", ._pd, .mul, .dst0x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp3x, .dst0x, ._, ._ }, + .{ ._, ._sd, .mul, .dst0x, .tmp3x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0q, .add_unaligned_size, -8), ._, ._, ._ }, + .{ .@"0:", .f_, .mul, .memi(.src0q, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .memad(.src0t, .add_unaligned_size, -16), ._, ._, ._ }, + .{ .@"0:", .f_, .ld, .memi(.src0t, .tmp0), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .tbyte }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .x87 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .mem(.src0t), ._, ._, ._ }, + .{ ._, .fn_cw, .st, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memd(.tmp1b, 1), ._, ._ }, + .{ ._, ._, .@"or", .memd(.tmp1b, 1), .ui(0b000_0_00_11), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp1b, 1), .tmp2b, ._, ._ }, + .{ ._, ._, .mov, .tmp2p, .sia(16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp2, .add_unaligned_size), ._, ._, ._ }, + .{ ._, .f_p, .mul, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp2p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .float = .xword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .dst0x, .memad(.src0x, .add_unaligned_size, -16), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + } }, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s}.{s} {f} {f}", .{ + @tagName(air_tag), + @tagName(reduce.operation), + cg.typeOf(reduce.operand).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{reduce.operand}, &ops, cg); + }, + .splat => |air_tag| fallback: { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + if (cg.typeOf(ty_op.operand).toIntern() == .bool_type) break :fallback try cg.airSplat(inst); + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + var res: [1]Temp = undefined; + cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .broadcast, .dst0x, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .src0x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, .p_d, .shuf, .dst0x, .dst0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_b, .broadcast, .dst0y, .src0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .vp_b, .broadcast, .tmp1y, .src0b, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .tmp0x, .tmp0x, .tmp0x, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .tmp0x, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp1), .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .tmp0x, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp1), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp1d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, .p_d, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .memi(.dst0b, .tmp0), .src0b, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any }, + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-1, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .memi(.dst0b, .tmp0), .src0b, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .broadcast, .dst0x, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, .p_d, .shuf, .dst0x, .dst0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .broadcast, .dst0y, .src0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .vp_w, .broadcast, .tmp1y, .src0w, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, + .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_b, .shuf, .tmp2x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp2x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .ssse3, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .pshufb_splat_mem = .{ .size = .word } } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_b, .shuf, .src0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .shufl, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_d, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .word, .is = .word } }, .any }, + .src_constraints = .{ .{ .int_or_float = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .memi(.dst0w, .tmp0), .src0w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .broadcast, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .broadcast, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .broadcast, .dst0x, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b00_00_00_00) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b00_00_00_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .broadcast, .dst0y, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .broadcast, .dst0y, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .vp_d, .broadcast, .tmp1y, .src0d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp1y, .src0d, ._, ._ }, + .{ .@"0:", .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .v_ss, .broadcast, .tmp1x, .src0d, ._, ._ }, + .{ .@"0:", .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .vp_d, .shuf, .tmp1x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .v_ps, .shuf, .tmp1x, .src0x, .src0x, .ui(0b00_00_00_00) }, + .{ .@"0:", .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._ps, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._ps, .shuf, .src0x, .src0x, .ui(0b00_00_00_00), ._ }, + .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .dword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .memi(.dst0d, .tmp0), .src0d, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .broadcast, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .movddup, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_, .movddup, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movddup, .dst0x, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b01_00_01_00), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .shuf, .dst0x, .src0x, .ui(0b0_0), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .sse, .none, .none } }, + }, + .dst_temps = .{ .{ .rc = .sse }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .shuf, .dst0x, .src0x, .ui(0b01_00_01_00), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._pd, .shuf, .dst0x, .src0x, .ui(0b0_0), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01_00_01_00), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_q, .broadcast, .dst0y, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .broadcast, .dst0y, .src0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .yword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp1y, .src0q, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_sd, .broadcast, .tmp1y, .src0q, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .v_, .movddup, .tmp1x, .src0q, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._, .movddup, .tmp1x, .src0q, ._, ._ }, + .{ .@"0:", ._pd, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b01_00_01_00), ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._pd, .shuf, .src0x, .src0x, .ui(0b0_0), ._ }, + .{ .@"0:", ._pd, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .p_d, .shuf, .tmp1x, .src0x, .ui(0b01_00_01_00), ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._pd, .shuf, .src0x, .src0x, .ui(0b0_0), ._ }, + .{ .@"0:", ._pd, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._ps, .shuf, .src0x, .src0x, .ui(0b01_00_01_00), ._ }, + .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._, .mov, .memi(.dst0q, .tmp0), .src0q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .broadcast, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .broadcast, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u128, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_i128, .broadcast, .tmp1y, .src0x, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u128, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_i128, .insert, .tmp1y, .src0y, .src0x, .ui(1) }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_f128, .broadcast, .tmp1y, .src0x, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_f128, .insert, .tmp1y, .src0y, .src0x, .ui(1) }, + .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{ .{ .multiple_scalar = .{ .of = .xword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .int_or_float = .xword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .src0x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx2, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .broadcast, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx2, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_i128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .broadcast, .dst0y, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .dst_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_f128, .insert, .dst0y, .src0y, .src0x, .ui(1) }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx2, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u128, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_i128, .broadcast, .tmp1y, .src0x, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .dst0, .add_size), ._, ._ }, + .{ ._, .v_f128, .broadcast, .tmp1y, .src0x, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .mem(.src0x), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .mem(.src0x), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .float = .tbyte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .mem(.src0x), ._, ._ }, + .{ .@"0:", ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._nb, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .slow_incdec, null, null }, + .dst_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2q, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._a, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .dst_constraints = .{ .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .reg = .rdi } }, + .{ .type = .usize, .kind = .{ .reg = .rsi } }, + .{ .type = .usize, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_len), ._, ._ }, + .{ ._, ._, .lea, .tmp1q, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp2q, .mem(.src0), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + + .memset => try cg.airMemset(inst, false), + .memset_safe => try cg.airMemset(inst, true), + .memcpy, .memmove => |air_tag| { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }) ++ .{undefined}; + ops[2] = ops[0].getByteLen(cg) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + cg.typeOf(bin_op.rhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try ops[0].toSlicePtr(cg); + cg.select(&.{}, &.{}, &ops, switch (air_tag) { + else => unreachable, + inline .memcpy, .memmove => |symbol| comptime &.{.{ + .patterns = &.{ + .{ .src = .{ + .{ .to_param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .{ .to_param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } }, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = @tagName(symbol) } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }}, + }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + cg.typeOf(bin_op.rhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + ops[2].tracking(cg), + }), + else => |e| return e, + }; + for (ops) |op| try op.die(cg); + }, + .cmpxchg_weak, .cmpxchg_strong => try cg.airCmpxchg(inst), + .atomic_load => try cg.airAtomicLoad(inst), + .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered), + .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic), + .atomic_store_release => try cg.airAtomicStore(inst, .release), + .atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst), + .atomic_rmw => try cg.airAtomicRmw(inst), + .is_named_enum_value => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"32" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0p, .src0p, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"32" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0p, .src0p, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"8" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0p, .src0p, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"8" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .tag_name => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{.slice_const_u8_sentinel_0}, &ops, comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"32" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .zigcc, .after = 1, .at = 1 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"32" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p0, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .zigcc, .after = 1, .at = 1 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p0, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"8" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr = .{ .cc = .zigcc, .after = 1, .at = 1 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any_int, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"8" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .dst0p0, .mem(.src0), ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + if (res[0].tracking(cg).short != .register_pair) try ops[0].toPair(&res[0], cg); + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .error_name => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [2]Temp = undefined; + cg.select(&res, &.{ .slice_const_u8_sentinel_0, .usize }, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp1d, .src0b, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .lea(.tmp0), ._, ._ }, + .{ ._, ._, .mov, .dst1d, .leasid(.dst0d, .@"4", .tmp1, (2 - 1) * 4), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .leasid(.dst0d, .@"4", .tmp1, (1 - 1) * 4), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .not, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst1d, .leai(.dst1, .tmp1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp1d, .src0w, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .lea(.tmp0), ._, ._ }, + .{ ._, ._, .mov, .dst1d, .leasid(.dst0d, .@"4", .tmp1, (2 - 1) * 4), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .leasid(.dst0d, .@"4", .tmp1, (1 - 1) * 4), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .not, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst1d, .leai(.dst1, .tmp1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, + .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .rc = .general_purpose } }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp1d, .src0d, ._, ._ }, + .{ ._, ._, .lea, .dst0p, .lea(.tmp0), ._, ._ }, + .{ ._, ._, .mov, .dst1d, .leasid(.dst0d, .@"4", .tmp1, (2 - 1) * 4), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .leasid(.dst0d, .@"4", .tmp1, (1 - 1) * 4), ._, ._ }, + .{ ._, ._, .lea, .dst0p, .leai(.dst0, .tmp1), ._, ._ }, + .{ ._, ._, .not, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .lea, .dst1d, .leai(.dst1, .tmp1), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(un_op).fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + const orig_res = res; + try res[0].toPair(&res[1], cg); + for (&ops) |*op| for (orig_res) |orig_r| { + if (op.index != orig_r.index) continue; + op.* = res[0]; + break; + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .error_set_has_value => |air_tag| { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}) ++ .{try cg.tempInit(ty_op.ty.toType(), .none)}; + var res: [1]Temp = undefined; + cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"32" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src1 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src1 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .gpr }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_param_gpr = .{ .cc = .zigcc, .after = 0, .at = 0 } }, .none, .none } }, + }, + .call_frame = .{ .alignment = .@"8" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .lazy_sym = .{ .kind = .code, .ref = .src1 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .nz }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .zigcc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .src0d, .src0d, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{ + @tagName(air_tag), + ty_op.ty.toType().fmt(pt), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + for (ops[1..]) |op| try op.die(cg); + try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg); + }, + .aggregate_init => |air_tag| fallback: { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const agg_ty = ty_pl.ty.toType(); + if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or + (agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst); + var res = try cg.tempAllocMem(agg_ty); + const reset_index = cg.next_temp_index; + var bt = cg.liveness.iterateBigTomb(inst); + switch (ip.indexToKey(agg_ty.toIntern())) { + inline .array_type, .vector_type => |sequence_type| { + const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..@intCast(sequence_type.len)]); + const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu); + var elem_disp: u31 = 0; + for (elems) |elem_ref| { + var elem = try cg.tempFromOperand(elem_ref, bt.feed()); + try res.write(&elem, .{ .disp = elem_disp }, cg); + try elem.die(cg); + try cg.resetTemps(reset_index); + elem_disp += @intCast(elem_size); + } + if (@hasField(@TypeOf(sequence_type), "sentinel") and sequence_type.sentinel != .none) { + var sentinel = try cg.tempFromValue(.fromInterned(sequence_type.sentinel)); + try res.write(&sentinel, .{ .disp = elem_disp }, cg); + try sentinel.die(cg); + } + }, + .struct_type => { + const loaded_struct = ip.loadStructType(agg_ty.toIntern()); + const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]); + switch (loaded_struct.layout) { + .auto, .@"extern" => { + for (elems, 0..) |elem_ref, field_index| { + const elem_dies = bt.feed(); + if (loaded_struct.fieldIsComptime(ip, field_index)) continue; + if (!hack_around_sema_opv_bugs or Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]).hasRuntimeBitsIgnoreComptime(zcu)) { + var elem = try cg.tempFromOperand(elem_ref, elem_dies); + try res.write(&elem, .{ .disp = @intCast(loaded_struct.offsets.get(ip)[field_index]) }, cg); + try elem.die(cg); + try cg.resetTemps(reset_index); + } + } + }, + .@"packed" => return cg.fail("failed to select {s} {f}", .{ + @tagName(air_tag), + agg_ty.fmt(pt), + }), + } + }, + .tuple_type => |tuple_type| { + const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]); + var elem_disp: u31 = 0; + for (elems, 0..) |elem_ref, field_index| { + const elem_dies = bt.feed(); + if (tuple_type.values.get(ip)[field_index] != .none) continue; + const field_type = Type.fromInterned(tuple_type.types.get(ip)[field_index]); + elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp)); + var elem = try cg.tempFromOperand(elem_ref, elem_dies); + try res.write(&elem, .{ .disp = elem_disp }, cg); + try elem.die(cg); + try cg.resetTemps(reset_index); + elem_disp += @intCast(field_type.abiSize(zcu)); + } + }, + else => return cg.fail("failed to select {s} {f}", .{ + @tagName(air_tag), + agg_ty.fmt(pt), + }), + } + try res.finish(inst, &.{}, &.{}, cg); + }, + .union_init => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const union_init = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; + const union_ty = ty_pl.ty.toType(); + var ops = try cg.tempsFromOperands(inst, .{union_init.init}); + var res = try cg.tempAllocMem(union_ty); + const union_layout = union_ty.unionGetLayout(zcu); + if (union_layout.tag_size > 0) { + var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex( + union_ty.unionTagTypeSafety(zcu).?, + union_init.field_index, + )); + try res.write(&tag_temp, .{ + .disp = @intCast(union_layout.tagOffset()), + }, cg); + try tag_temp.die(cg); + } + try res.write(&ops[0], .{ + .disp = @intCast(union_layout.payloadOffset()), + }, cg); + try res.finish(inst, &.{union_init.init}, &ops, cg); + }, + .prefetch => { + const prefetch = air_datas[@intFromEnum(inst)].prefetch; + var ops = try cg.tempsFromOperands(inst, .{prefetch.ptr}); + switch (prefetch.cache) { + .instruction => {}, // prefetchi requires rip-relative addressing, which is currently non-trivial to emit from an arbitrary ptr value + .data => if (prefetch.rw == .write and prefetch.locality <= 2 and cg.hasFeature(.prefetchwt1)) { + try ops[0].toSlicePtr(cg); + while (try ops[0].toLea(cg)) {} + try cg.asmMemory(.{ ._wt1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); + } else if (prefetch.rw == .write and cg.hasFeature(.prfchw)) { + try ops[0].toSlicePtr(cg); + while (try ops[0].toLea(cg)) {} + try cg.asmMemory(.{ ._w, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); + } else if (cg.hasFeature(.sse) or cg.hasFeature(.prfchw) or cg.hasFeature(.prefetchi) or cg.hasFeature(.prefetchwt1)) { + try ops[0].toSlicePtr(cg); + while (try ops[0].toLea(cg)) {} + switch (prefetch.locality) { + 0 => try cg.asmMemory(.{ ._nta, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), + 1 => try cg.asmMemory(.{ ._t2, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), + 2 => try cg.asmMemory(.{ ._t1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), + 3 => try cg.asmMemory(.{ ._t0, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), + } + }, + } + const res = try cg.tempInit(.void, .none); + try res.finish(inst, &.{prefetch.ptr}, &ops, cg); + }, + .mul_add => |air_tag| { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs, pl_op.operand }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .required_features = .{ .fma, .f16c, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .src2q, ._, ._ }, + .{ ._, .v_ss, .fmadd213, .dst0x, .tmp0x, .tmp1d, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fma, .f16c, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .mem } }, + .{ .src = .{ .to_sse, .mem, .mem } }, + .{ .src = .{ .mem, .to_sse, .mem } }, + .{ .src = .{ .mem, .mem, .to_sse } }, + .{ .src = .{ .to_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_sse, .to_sse } }, + .{ .src = .{ .to_sse, .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_4_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1x, .src2q, ._, ._ }, + .{ ._, .v_ps, .fmadd213, .dst0x, .tmp0x, .tmp1x, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .fma, .f16c, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .mem } }, + .{ .src = .{ .to_sse, .mem, .mem } }, + .{ .src = .{ .mem, .to_sse, .mem } }, + .{ .src = .{ .mem, .mem, .to_sse } }, + .{ .src = .{ .to_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_sse, .to_sse } }, + .{ .src = .{ .to_sse, .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp1y, .src2x, ._, ._ }, + .{ ._, .v_ps, .fmadd213, .dst0y, .tmp0y, .tmp1y, ._ }, + .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ }, + } }, + }, .{ + .required_features = .{ .fma, .f16c, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .{ .type = .vector_8_f16, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp2y, .memi(.src1x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memi(.src2x, .tmp0), ._, ._ }, + .{ ._, .v_ps, .fmadd213, .tmp1y, .tmp2y, .tmp3y, ._ }, + .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp3x, .memi(.src0w, .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memi(.src1w, .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memi(.src2w, .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memi(.src1w, .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memi(.src2w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp1x, .ui(0), ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memi(.src1w, .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memi(.src2w, .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .p_w, .extr, .tmp5d, .tmp1x, .ui(0), ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp5w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .ax } }, + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmah" } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memi(.src0w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memi(.src1w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memi(.src2w, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ }, + .{ ._, ._ss, .mov, .tmp5x, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .call, .tmp6d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp1w, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .fmadd132, .dst0x, .src2x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .fmadd213, .dst0x, .src1x, .src2d, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, + .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, + }, + .dst_temps = .{ .{ .ref = .src2 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .fmadd231, .dst0x, .src0x, .src1d, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmaf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .fmadd132, .dst0x, .src2x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .fmadd213, .dst0x, .src1x, .src2x, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, + .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, + }, + .dst_temps = .{ .{ .ref = .src2 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .fmadd231, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .fmadd132, .dst0y, .src2y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .fmadd213, .dst0y, .src1y, .src2y, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, + .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, + }, + .dst_temps = .{ .{ .ref = .src2 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_ps, .fmadd231, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .memi(.src1y, .tmp0), ._, ._ }, + .{ ._, .v_ps, .fmadd213, .tmp1y, .tmp2y, .memi(.src2y, .tmp0), ._ }, + .{ ._, .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, .v_ss, .mov, .tmp2x, .memi(.src1d, .tmp0), ._, ._ }, + .{ ._, .v_ss, .mov, .tmp3x, .memi(.src2d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f32, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaf" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ss, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ }, + .{ ._, ._ss, .mov, .tmp2x, .memi(.src1d, .tmp0), ._, ._ }, + .{ ._, ._ss, .mov, .tmp3x, .memi(.src2d, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .fmadd132, .dst0x, .src2x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .fmadd213, .dst0x, .src1x, .src2q, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, + .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, + }, + .dst_temps = .{ .{ .ref = .src2 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .fmadd231, .dst0x, .src0x, .src1q, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .fmadd132, .dst0x, .src2x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .fmadd213, .dst0x, .src1x, .src2x, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, + .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, + }, + .dst_temps = .{ .{ .ref = .src2 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .fmadd231, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem, .to_sse } }, + .{ .src = .{ .mem, .to_mut_sse, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .fmadd132, .dst0y, .src2y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .to_sse, .mem } }, + .{ .src = .{ .to_sse, .to_mut_sse, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mut_sse, .sse, .sse } }, + .{ .src = .{ .sse, .mut_sse, .sse }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .fmadd213, .dst0y, .src1y, .src2y, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem, .to_mut_sse } }, + .{ .src = .{ .mem, .to_sse, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse, .to_mut_sse } }, + }, + .dst_temps = .{ .{ .ref = .src2 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .fmadd231, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .fma, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } }, + .{ .type = .vector_4_f64, .kind = .{ .mut_rc = .{ .ref = .src2, .rc = .sse } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp1y, .memi(.src0y, .tmp0), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .memi(.src1y, .tmp0), ._, ._ }, + .{ ._, .v_pd, .fmadd213, .tmp1y, .tmp2y, .memi(.src2y, .tmp0), ._ }, + .{ ._, .v_pd, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, .v_sd, .mov, .tmp2x, .memi(.src1q, .tmp0), ._, ._ }, + .{ ._, .v_sd, .mov, .tmp3x, .memi(.src2q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._sd, .mov, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._sd, .mov, .tmp2x, .memi(.src1q, .tmp0), ._, ._ }, + .{ ._, ._sd, .mov, .tmp3x, .memi(.src2q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fma" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, ._ps, .movl, .tmp1x, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._ps, .movl, .tmp2x, .memi(.src1q, .tmp0), ._, ._ }, + .{ ._, ._ps, .movl, .tmp3x, .memi(.src2q, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .size = 16 * 3, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .reg = .st0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._ps, .mova, .tmp0x, .mem(.src0x), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp1x), .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .mem(.src2x), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp1x, 16 * 2), .tmp0x, ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .mem(.src2), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .required_features = .{ .sse, .x87, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .size = 16 * 3, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f80, .kind = .{ .frame = .call_frame } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .memi(.src1x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp2x, 16), .tmp1x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .memi(.src2x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .memd(.tmp2x, 16 * 2), .tmp1x, ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ }, + .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__fmax" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memi(.src1, .tmp0), ._, ._ }, + .{ ._, ._, .lea, .tmp4p, .memi(.src2, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } }, + .{ .to_param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } }, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memi(.src2x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memi(.src2x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memi(.src2x, .tmp0), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ }, + .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + ops[2].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand }, &ops, cg); + }, + .field_parent_ptr => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const field_parent_ptr = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + var ops = try cg.tempsFromOperands(inst, .{field_parent_ptr.field_ptr}); + try ops[0].toOffset(-@as(i32, @intCast(codegen.fieldOffset( + ty_pl.ty.toType(), + cg.typeOf(field_parent_ptr.field_ptr), + field_parent_ptr.field_index, + zcu, + ))), cg); + try ops[0].finish(inst, &.{field_parent_ptr.field_ptr}, &ops, cg); + }, + .wasm_memory_size, .wasm_memory_grow => unreachable, + .cmp_lt_errors_len => |air_tag| { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + var res: [1]Temp = undefined; + cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .be }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .lea(.tmp0), ._, ._ }, + .{ ._, ._, .cmp, .src0b, .lea(.tmp1b), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .be }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .lea(.tmp0), ._, ._ }, + .{ ._, ._, .cmp, .src0w, .lea(.tmp1w), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .anyerror, .kind = .{ .lazy_sym = .{ .kind = .const_data } } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .be }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp1p, .lea(.tmp0), ._, ._ }, + .{ ._, ._, .cmp, .src0d, .lea(.tmp1d), ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {f}", .{ + @tagName(air_tag), + ops[0].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{un_op}, &ops, cg); + }, + .err_return_trace => { + const ert: Temp = .{ .index = err_ret_trace_index }; + try ert.finish(inst, &.{}, &.{}, cg); + }, + .set_err_return_trace => { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try cg.tempsFromOperands(inst, .{un_op}); + switch (ops[0].unwrap(cg)) { + .ref => { + const result = try cg.allocRegOrMem(err_ret_trace_index, true); + try cg.genCopy(.usize, result, ops[0].tracking(cg).short, .{}); + tracking_log.debug("{f} => {f} (birth)", .{ err_ret_trace_index, result }); + cg.inst_tracking.putAssumeCapacityNoClobber(err_ret_trace_index, .init(result)); + }, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + tracking_log.debug("{f} => {f} (birth)", .{ err_ret_trace_index, temp_tracking.short }); + cg.inst_tracking.putAssumeCapacityNoClobber(err_ret_trace_index, temp_tracking.*); + assert(cg.reuseTemp(err_ret_trace_index, temp_index.toIndex(), temp_tracking)); + }, + .err_ret_trace => unreachable, + } + }, + .addrspace_cast => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); + }, + .save_err_return_trace_index => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const agg_ty = ty_pl.ty.toType(); + assert(agg_ty.containerLayout(zcu) != .@"packed"); + var ert: Temp = .{ .index = err_ret_trace_index }; + var res = try ert.load(.usize, .{ .disp = @intCast(agg_ty.structFieldOffset(ty_pl.payload, zcu)) }, cg); + try ert.die(cg); + try res.finish(inst, &.{}, &.{}, cg); + }, + .vector_store_elem => { + const vector_store_elem = air_datas[@intFromEnum(inst)].vector_store_elem; + const bin_op = cg.air.extraData(Air.Bin, vector_store_elem.payload).data; + var ops = try cg.tempsFromOperands(inst, .{ vector_store_elem.vector_ptr, bin_op.lhs, bin_op.rhs }); + cg.select(&.{}, &.{}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._s, .bt, .tmp0d, .src1d, ._, ._ }, + .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._s, .bt, .lea(.src0w), .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .ptr_bool_vec = .dword }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .lea(.src0d), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, + .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, + .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0d), .tmp0d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .ptr_bool_vec = .qword }, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .lea(.src0q), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, + .{ ._, ._r, .bt, .tmp1q, .src1q, ._, ._ }, + .{ ._, ._s, .bt, .tmp0q, .src1q, ._, ._ }, + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .lea(.src0q), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp0d, .ui(5), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .leasi(.src0d, .@"4", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, + .{ ._, ._r, .bt, .tmp2d, .src1d, ._, ._ }, + .{ ._, ._s, .bt, .tmp1d, .src1d, ._, ._ }, + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._z, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .leasi(.src0d, .@"4", .tmp0), .tmp1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, + .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, + .{ .@"1:", ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any, .any, .{ .int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .imm8 } }, + .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leaa(.src0b, .add_src0_elem_size_mul_src1), .src2b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any, .any, .{ .int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .imm8 } }, + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leai(.src0b, .src1), .src2b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any, .any, .{ .int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .imm16 } }, + .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any, .any, .{ .int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .imm16 } }, + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .each = .{ .once = &.{ + .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, + .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, + .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .each = .{ .once = &.{ + .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, + .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f32, .kind = .mem }, + .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, + .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any, .any, .{ .int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .imm32 } }, + .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .any, .any, .{ .int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .imm32 } }, + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any, .any, .{ .int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .simm32 } }, + .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any, .any, .{ .int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .simm32 } }, + .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, + }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .simm32, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .any, .any, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, + }, + .each = .{ .once = &.{ + .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => { + const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu); + while (try ops[0].toRegClass(true, .general_purpose, cg) or + try ops[1].toRegClass(true, .general_purpose, cg)) + {} + const base_reg = ops[0].tracking(cg).short.register.to64(); + const rhs_reg = ops[1].tracking(cg).short.register.to64(); + if (!std.math.isPowerOfTwo(elem_size)) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + rhs_reg, + rhs_reg, + .u(elem_size), + ); + try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ + .base = .{ .reg = base_reg }, + .mod = .{ .rm = .{ .index = rhs_reg } }, + }); + } else if (elem_size > 8) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterImmediate( + .{ ._l, .sh }, + rhs_reg, + .u(std.math.log2_int(u64, elem_size)), + ); + try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ + .base = .{ .reg = base_reg }, + .mod = .{ .rm = .{ .index = rhs_reg } }, + }); + } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ + .base = .{ .reg = base_reg }, + .mod = .{ .rm = .{ + .index = rhs_reg, + .scale = .fromFactor(@intCast(elem_size)), + } }, + }); + try ops[0].store(&ops[2], .{}, cg); + }, + else => |e| return e, + }; + for (ops) |op| try op.die(cg); + }, + .runtime_nav_ptr => { + const ty_nav = air_datas[@intFromEnum(inst)].ty_nav; + const nav = ip.getNav(ty_nav.nav); + const is_threadlocal = zcu.comp.config.any_non_single_threaded and nav.isThreadlocal(ip); + if (is_threadlocal) if (cg.mod.pic) { + try cg.spillRegisters(&.{ .rdi, .rax }); + } else { + try cg.spillRegisters(&.{.rax}); + }; + var res = try cg.tempInit(.fromInterned(ty_nav.ty), .{ .lea_nav = ty_nav.nav }); + if (is_threadlocal) while (try res.toRegClass(true, .general_purpose, cg)) {}; + try res.finish(inst, &.{}, &.{}, cg); + }, + .c_va_arg => try cg.airVaArg(inst), + .c_va_copy => try cg.airVaCopy(inst), + .c_va_end => try cg.airVaEnd(inst), + .c_va_start => try cg.airVaStart(inst), + .work_item_id, .work_group_size, .work_group_id => unreachable, + } + try cg.resetTemps(@enumFromInt(0)); + cg.checkInvariantsAfterAirInst(); + } + verbose_tracking_log.debug("{f}", .{cg.fmtTracking()}); +} + +fn genLazy(cg: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + switch (ip.indexToKey(lazy_sym.ty)) { + .enum_type => { + const enum_ty: Type = .fromInterned(lazy_sym.ty); + wip_mir_log.debug("{f}.@tagName:", .{enum_ty.fmt(pt)}); + + const ret_regs = abi.getCAbiIntReturnRegs(.auto)[0..2].*; + const ret_locks = cg.register_manager.lockRegsAssumeUnused(2, ret_regs); + defer for (ret_locks) |lock| cg.register_manager.unlockReg(lock); + + const param_reg = abi.getCAbiIntParamRegs(.auto)[0]; + const param_lock = cg.register_manager.lockReg(param_reg); + defer if (param_lock) |lock| cg.register_manager.unlockReg(lock); + + const ret_mcv: MCValue = .{ .register_pair = ret_regs }; + + const data_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); + const data_lock = cg.register_manager.lockRegAssumeUnused(data_reg); + defer cg.register_manager.unlockReg(data_lock); + try cg.asmRegisterMemory(.{ ._, .lea }, data_reg.to64(), .{ + .base = .{ .lazy_sym = .{ .kind = .const_data, .ty = lazy_sym.ty } }, + }); + + var data_off: i32 = 0; + const reset_index = cg.next_temp_index; + const tag_names = ip.loadEnumType(lazy_sym.ty).names; + for (0..tag_names.len) |tag_index| { + var enum_temp = try cg.tempInit(enum_ty, if (enum_ty.abiSize(zcu) <= @as(u4, switch (cg.target.cpu.arch) { + else => unreachable, + .x86 => 4, + .x86_64 => 8, + })) .{ .register = param_reg } else .{ .indirect = .{ .reg = param_reg } }); + + const tag_name_len = tag_names.get(ip)[tag_index].length(ip); + var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index))); + const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, cg) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try enum_temp.die(cg); + try tag_temp.die(cg); + const skip_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); + try cc_temp.die(cg); + + try cg.genSetReg( + ret_mcv.register_pair[0], + .usize, + .{ .register_offset = .{ .reg = data_reg, .off = data_off } }, + .{}, + ); + try cg.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{}); + try cg.asmOpOnly(.{ ._, .ret }); + + cg.performReloc(skip_reloc); + + data_off += @intCast(tag_name_len + 1); + + try cg.resetTemps(reset_index); + } + + try cg.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{}); + try cg.asmOpOnly(.{ ._, .ret }); + }, + .error_set_type => |error_set_type| { + const err_ty: Type = .fromInterned(lazy_sym.ty); + wip_mir_log.debug("{f}.@errorCast:", .{err_ty.fmt(pt)}); + + const ret_reg = abi.getCAbiIntReturnRegs(.auto)[0]; + const ret_lock = cg.register_manager.lockRegAssumeUnused(ret_reg); + defer cg.register_manager.unlockReg(ret_lock); + + const param_reg = abi.getCAbiIntParamRegs(.auto)[0]; + const param_lock = cg.register_manager.lockReg(param_reg); + defer if (param_lock) |lock| cg.register_manager.unlockReg(lock); + + const ret_mcv: MCValue = .{ .register = ret_reg }; + const err_mcv: MCValue = .{ .register = param_reg }; + var err_temp = try cg.tempInit(err_ty, err_mcv); + + const ExpectedContents = [32]Mir.Inst.Index; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); + const allocator = stack.get(); + + const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len); + defer allocator.free(relocs); + + const reset_index = cg.next_temp_index; + for (0.., relocs) |tag_index, *reloc| { + var tag_temp = try cg.tempInit(.anyerror, .{ + .immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?, + }); + const cc_temp = err_temp.cmpInts(.eq, &tag_temp, cg) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try tag_temp.die(cg); + reloc.* = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); + try cc_temp.die(cg); + try cg.resetTemps(reset_index); + } + try err_temp.die(cg); + + try cg.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{}); + for (relocs) |reloc| cg.performReloc(reloc); + assert(ret_mcv.register == err_mcv.register); + try cg.asmOpOnly(.{ ._, .ret }); + }, + else => return cg.fail( + "TODO implement {s} for {f}", + .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, + ), + } + try cg.resetTemps(@enumFromInt(0)); + cg.checkInvariantsAfterAirInst(); +} + +fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { + for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst); + switch (value) { + else => {}, + .eflags, .register_overflow => self.eflags_inst = inst, + } +} + +fn getValueIfFree(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) void { + for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg)) + self.register_manager.getRegAssumeFree(reg, inst); +} + +const FreeOptions = struct { emit_instructions: bool = true }; + +fn freeReg(self: *CodeGen, reg: Register, comptime opts: FreeOptions) !void { + self.register_manager.freeReg(reg); + if (opts.emit_instructions and reg.isClass(.x87)) try self.asmRegister(.{ .f_, .free }, reg); +} + +fn freeValue(self: *CodeGen, value: MCValue, comptime opts: FreeOptions) !void { + switch (value) { + .register => |reg| try self.freeReg(reg, opts), + inline .register_pair, + .register_triple, + .register_quadruple, + => |regs| for (regs) |reg| try self.freeReg(reg, opts), + .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg, opts), + .register_overflow => |reg_ov| { + try self.freeReg(reg_ov.reg, opts); + self.eflags_inst = null; + }, + .register_mask => |reg_mask| try self.freeReg(reg_mask.reg, opts), + .eflags => self.eflags_inst = null, + else => {}, // TODO process stack allocation death + } +} + +fn feed(self: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void { + if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst, .{}); +} + +/// Asserts there is already capacity to insert into top branch inst_table. +fn processDeath(self: *CodeGen, inst: Air.Inst.Index, comptime opts: FreeOptions) !void { + try self.inst_tracking.getPtr(inst).?.die(self, inst, opts); +} + +fn finishAirResult(self: *CodeGen, inst: Air.Inst.Index, result: MCValue) void { + if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) { + .none, .dead, .unreach => {}, + else => unreachable, // Why didn't the result die? + } else { + tracking_log.debug("{f} => {f} (birth)", .{ inst, result }); + self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result)); + // In some cases, an operand may be reused as the result. + // If that operand died and was a register, it was freed by + // processDeath, so we have to "re-allocate" the register. + self.getValueIfFree(result, inst); + } +} + +fn finishAir( + self: *CodeGen, + inst: Air.Inst.Index, + result: MCValue, + operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, +) !void { + const tomb_bits = self.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (self.reused_operands.isSet(op_index)) continue; + try self.processDeath(op.toIndexAllowNone() orelse continue, .{}); + } + self.finishAirResult(inst, result); +} + +const FrameLayout = struct { + stack_mask: u32, + stack_adjust: u32, + save_reg_list: Mir.RegisterList, +}; + +fn setFrameLoc( + self: *CodeGen, + frame_index: FrameIndex, + base: Register, + offset: *i32, + comptime aligned: bool, +) void { + const frame_i = @intFromEnum(frame_index); + if (aligned) { + const alignment = self.frame_allocs.items(.abi_align)[frame_i]; + offset.* = @intCast(alignment.forward(@intCast(offset.*))); + } + self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* }); + offset.* += self.frame_allocs.items(.abi_size)[frame_i]; +} + +fn computeFrameLayout(self: *CodeGen, cc: std.builtin.CallingConvention.Tag) !FrameLayout { + const frame_allocs_len = self.frame_allocs.len; + try self.frame_locs.resize(self.gpa, frame_allocs_len); + const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count); + defer self.gpa.free(stack_frame_order); + + const frame_size = self.frame_allocs.items(.abi_size); + const frame_align = self.frame_allocs.items(.abi_align); + const frame_offset = self.frame_locs.items(.disp); + + for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index| + frame_order.* = @enumFromInt(frame_index); + { + const SortContext = struct { + frame_align: @TypeOf(frame_align), + pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool { + return context.frame_align[@intFromEnum(lhs)].compare(.gt, context.frame_align[@intFromEnum(rhs)]); + } + }; + const sort_context = SortContext{ .frame_align = frame_align }; + std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan); + } + + const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)]; + const stack_frame_align = frame_align[@intFromEnum(FrameIndex.stack_frame)]; + const args_frame_align = frame_align[@intFromEnum(FrameIndex.args_frame)]; + const needed_align = call_frame_align.max(stack_frame_align); + const need_align_stack = needed_align.compare(.gt, args_frame_align); + + // Create list of registers to save in the prologue. + // TODO handle register classes + var save_reg_list: Mir.RegisterList = .empty; + const callee_preserved_regs = abi.getCalleePreservedRegs(cc); + for (callee_preserved_regs) |reg| { + if (self.register_manager.isRegAllocated(reg)) { + save_reg_list.push(callee_preserved_regs, reg); + } + } + + var rbp_offset: i32 = 0; + self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false); + self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false); + self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false); + const stack_frame_align_offset = if (need_align_stack) + 0 + else + save_reg_list.size(self.target) + frame_offset[@intFromEnum(FrameIndex.args_frame)]; + + var rsp_offset: i32 = 0; + self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true); + self.setFrameLoc(.stack_frame, .rsp, &rsp_offset, true); + for (stack_frame_order) |frame_index| self.setFrameLoc(frame_index, .rsp, &rsp_offset, true); + rsp_offset += stack_frame_align_offset; + rsp_offset = @intCast(needed_align.forward(@intCast(rsp_offset))); + rsp_offset -= stack_frame_align_offset; + frame_size[@intFromEnum(FrameIndex.call_frame)] = + @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]); + + return .{ + .stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0), + .stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]), + .save_reg_list = save_reg_list, + }; +} + +fn getFrameAddrAlignment(self: *CodeGen, frame_addr: bits.FrameAddr) InternPool.Alignment { + const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align; + return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off))); +} + +fn getFrameAddrSize(self: *CodeGen, frame_addr: bits.FrameAddr) u32 { + return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off)); +} + +fn allocFrameIndex(self: *CodeGen, alloc: FrameAlloc) !FrameIndex { + const frame_allocs_slice = self.frame_allocs.slice(); + const frame_size = frame_allocs_slice.items(.abi_size); + const frame_align = frame_allocs_slice.items(.abi_align); + + const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)]; + stack_frame_align.* = stack_frame_align.max(alloc.abi_align); + + for (self.free_frame_indices.keys(), 0..) |frame_index, free_i| { + const abi_size = frame_size[@intFromEnum(frame_index)]; + if (abi_size != alloc.abi_size) continue; + const abi_align = &frame_align[@intFromEnum(frame_index)]; + abi_align.* = abi_align.max(alloc.abi_align); + + _ = self.free_frame_indices.swapRemoveAt(free_i); + return frame_index; + } + const frame_index: FrameIndex = @enumFromInt(self.frame_allocs.len); + try self.frame_allocs.append(self.gpa, alloc); + return frame_index; +} + +/// Use a pointer instruction as the basis for allocating stack memory. +fn allocMemPtr(self: *CodeGen, inst: Air.Inst.Index) !FrameIndex { + const pt = self.pt; + const zcu = pt.zcu; + const ptr_ty = self.typeOfIndex(inst); + const val_ty = ptr_ty.childType(zcu); + return self.allocFrameIndex(.init(.{ + .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse { + return self.fail("type '{f}' too big to fit into stack frame", .{val_ty.fmt(pt)}); + }, + .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"), + })); +} + +fn allocRegOrMem(self: *CodeGen, inst: Air.Inst.Index, reg_ok: bool) !MCValue { + return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok); +} + +fn allocTempRegOrMem(self: *CodeGen, elem_ty: Type, reg_ok: bool) !MCValue { + return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok); +} + +fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse { + return self.fail("type '{f}' too big to fit into stack frame", .{ty.fmt(pt)}); + }; + + if (reg_ok) need_mem: { + if (std.math.isPowerOfTwo(abi_size) and abi_size <= @as(u32, max_abi_size: switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(self.target)) { + 16, 32, 64, 128 => 16, + 80 => break :need_mem, + else => unreachable, + }, + .vector => { + const elem_ty = ty.childType(zcu); + break :max_abi_size if (elem_ty.toIntern() == .bool_type) + 8 + else if (self.floatBits(elem_ty)) |float_bits| switch (float_bits) { + 16, 32, 64, 128 => self.vectorSize(.float), + 80 => break :need_mem, + else => unreachable, + } else self.vectorSize(.int); + }, + else => 8, + })) { + if (self.register_manager.tryAllocReg(inst, self.regSetForType(ty))) |reg| { + return MCValue{ .register = registerAlias(reg, abi_size) }; + } + } + } + + const frame_index = try self.allocFrameIndex(.initSpill(ty, zcu)); + return .{ .load_frame = .{ .index = frame_index } }; +} + +fn regClassForType(self: *CodeGen, ty: Type) Register.Class { + const pt = self.pt; + const zcu = pt.zcu; + if (self.floatBits(ty)) |float_bits| return switch (float_bits) { + 80 => .x87, + else => .sse, + }; + if (!ty.isVector(zcu)) return .general_purpose; + const elem_ty = ty.childType(zcu); + return if (elem_ty.toIntern() == .bool_type) + .general_purpose + else if (self.floatBits(elem_ty)) |float_bits| + if (float_bits == 80) .x87 else .sse + else if (self.intInfo(elem_ty)) |_| + .sse + else + .general_purpose; +} + +fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet { + return switch (rc) { + .general_purpose => abi.RegisterClass.gp, + .gphi => abi.RegisterClass.gphi, + .segment, .ip, .cr, .dr => unreachable, + .x87 => abi.RegisterClass.x87, + .mmx => @panic("TODO"), + .sse => abi.RegisterClass.sse, + }; +} + +fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet { + return regSetForRegClass(self.regClassForType(ty)); +} + +fn vectorSize(cg: *CodeGen, kind: enum { int, float }) u6 { + return if (cg.hasFeature(switch (kind) { + .int => .avx2, + .float => .avx, + })) 32 else if (cg.hasFeature(.sse)) 16 else 8; +} + +const State = struct { + registers: RegisterManager.TrackedRegisters, + reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, + free_registers: RegisterManager.RegisterBitSet, + next_temp_index: Temp.Index, + inst_tracking_len: u32, + scope_generation: u32, +}; + +fn initRetroactiveState(self: *CodeGen) State { + const scope_generation = self.scope_generation + 1; + self.scope_generation = scope_generation; + + var state: State = undefined; + state.next_temp_index = @enumFromInt(0); + state.inst_tracking_len = @intCast(self.inst_tracking.count()); + state.scope_generation = scope_generation; + return state; +} + +fn saveRetroactiveState(self: *CodeGen, state: *State) !void { + try self.spillEflagsIfOccupied(); + const free_registers = self.register_manager.free_registers; + var it = free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = self.register_manager.registers[index]; + state.registers[index] = tracked_inst; + state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?; + } + state.free_registers = free_registers; +} + +fn saveState(self: *CodeGen) !State { + var state = self.initRetroactiveState(); + try self.saveRetroactiveState(&state); + return state; +} + +fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct { + emit_instructions: bool, + update_tracking: bool, + resurrect: bool, + close_scope: bool, +}) !void { + if (opts.close_scope) { + for ( + self.inst_tracking.keys()[@intFromEnum(state.next_temp_index)..@intFromEnum(self.next_temp_index)], + self.inst_tracking.values()[@intFromEnum(state.next_temp_index)..@intFromEnum(self.next_temp_index)], + ) |inst, *tracking| try tracking.die(self, inst, .{ .emit_instructions = opts.emit_instructions }); + self.next_temp_index = state.next_temp_index; + for ( + self.inst_tracking.keys()[state.inst_tracking_len..], + self.inst_tracking.values()[state.inst_tracking_len..], + ) |inst, *tracking| try tracking.die(self, inst, .{ .emit_instructions = opts.emit_instructions }); + self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len); + } + + if (opts.resurrect) { + for ( + self.inst_tracking.keys()[0..@intFromEnum(state.next_temp_index)], + self.inst_tracking.values()[0..@intFromEnum(state.next_temp_index)], + ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation); + for ( + self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len], + self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len], + ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation); + } + for (deaths) |death| try self.processDeath(death, .{ .emit_instructions = opts.emit_instructions }); + + const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + if (opts.update_tracking) {} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); + + var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity( + stack.get(), + @typeInfo(ExpectedContents).array.len, + ); + defer if (!opts.update_tracking) { + for (reg_locks.items) |lock| self.register_manager.unlockReg(lock); + reg_locks.deinit(); + }; + + for ( + 0.., + self.register_manager.registers, + state.registers, + state.reg_tracking, + ) |reg_i, current_slot, target_slot, reg_tracking| { + const reg_index: RegisterManager.TrackedIndex = @intCast(reg_i); + const current_maybe_inst = if (self.register_manager.isRegIndexFree(reg_index)) null else current_slot; + const target_maybe_inst = if (state.free_registers.isSet(reg_index)) null else target_slot; + if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst| + assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len); + if (opts.emit_instructions and current_maybe_inst != target_maybe_inst) { + if (current_maybe_inst) |current_inst| + try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst); + if (target_maybe_inst) |target_inst| + try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg_tracking); + } + if (opts.update_tracking) { + if (current_maybe_inst) |current_inst| { + try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst); + self.register_manager.freeRegIndex(reg_index); + } + if (target_maybe_inst) |target_inst| { + self.register_manager.getRegIndexAssumeFree(reg_index, target_inst); + self.inst_tracking.getPtr(target_inst).?.trackMaterialize(target_inst, reg_tracking); + } + } else if (target_maybe_inst) |_| + try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(reg_index)); + } + if (opts.emit_instructions) if (self.eflags_inst) |inst| + try self.inst_tracking.getPtr(inst).?.spill(self, inst); + if (opts.update_tracking) if (self.eflags_inst) |inst| { + self.eflags_inst = null; + try self.inst_tracking.getPtr(inst).?.trackSpill(self, inst); + }; + + if (opts.update_tracking and std.debug.runtime_safety) { + assert(self.eflags_inst == null); + assert(self.register_manager.free_registers.eql(state.free_registers)); + var used_reg_it = state.free_registers.iterator(.{ .kind = .unset }); + while (used_reg_it.next()) |index| + assert(self.register_manager.registers[index] == state.registers[index]); + } +} + +pub fn spillInstruction(self: *CodeGen, reg: Register, inst: Air.Inst.Index) !void { + const tracking = self.inst_tracking.getPtr(inst) orelse return; + for (tracking.getRegs()) |tracked_reg| { + if (tracked_reg.id() == reg.id()) break; + } else unreachable; // spilled reg not tracked with spilled instruction + try tracking.spill(self, inst); + try tracking.trackSpill(self, inst); +} + +pub fn spillEflagsIfOccupied(self: *CodeGen) !void { + if (self.eflags_inst) |inst| { + self.eflags_inst = null; + const tracking = self.inst_tracking.getPtr(inst).?; + assert(tracking.getCondition() != null); + try tracking.spill(self, inst); + try tracking.trackSpill(self, inst); + } +} + +pub fn spillCallerPreservedRegs(self: *CodeGen, cc: std.builtin.CallingConvention.Tag, ignore_reg: Register) !void { + switch (cc) { + inline .auto, .x86_64_sysv, .x86_64_win => |tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| + if (reg != ignore_reg) try self.register_manager.getKnownReg(reg, null), + else => unreachable, + } +} + +pub fn spillRegisters(self: *CodeGen, comptime registers: []const Register) !void { + inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null); +} + +/// Copies a value to a register without tracking the register. The register is not considered +/// allocated. A second call to `copyToTmpRegister` may return the same register. +/// This can have a side effect of spilling instructions to the stack to free up a register. +fn copyToTmpRegister(self: *CodeGen, ty: Type, mcv: MCValue) !Register { + const reg = try self.register_manager.allocReg(null, self.regSetForType(ty)); + try self.genSetReg(reg, ty, mcv, .{}); + return reg; +} + +/// Allocates a new register and copies `mcv` into it. +/// `reg_owner` is the instruction that gets associated with the register in the register table. +/// This can have a side effect of spilling instructions to the stack to free up a register. +/// WARNING make sure that the allocated register matches the returned MCValue from an instruction! +fn copyToRegisterWithInstTracking( + self: *CodeGen, + reg_owner: Air.Inst.Index, + ty: Type, + mcv: MCValue, +) !MCValue { + const reg: Register = try self.register_manager.allocReg(reg_owner, self.regSetForType(ty)); + try self.genSetReg(reg, ty, mcv, .{}); + return MCValue{ .register = reg }; +} + +fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void { + const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } }; + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const result: MCValue = switch (self.ret_mcv.long) { + else => unreachable, + .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } }, + .load_frame => .{ .register_offset = .{ + .reg = (try self.copyToRegisterWithInstTracking( + inst, + self.typeOfIndex(inst), + self.ret_mcv.long, + )).register, + .off = self.ret_mcv.short.indirect.off, + } }, + }; + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const dst_ty = self.typeOfIndex(inst); + const dst_bits = dst_ty.floatBits(self.target); + const src_ty = self.typeOf(ty_op.operand); + const src_bits = src_ty.floatBits(self.target); + + const result = result: { + if (switch (dst_bits) { + 16 => switch (src_bits) { + 32 => !self.hasFeature(.f16c), + 64, 80, 128 => true, + else => unreachable, + }, + 32 => switch (src_bits) { + 64 => false, + 80, 128 => true, + else => unreachable, + }, + 64 => switch (src_bits) { + 80, 128 => true, + else => unreachable, + }, + 80 => switch (src_bits) { + 128 => true, + else => unreachable, + }, + else => unreachable, + }) { + var sym_buf: ["__trunc?f?f2".len]u8 = undefined; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(), + .param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "__trunc{c}f{c}f2", .{ + floatCompilerRtAbiName(src_bits), + floatCompilerRtAbiName(dst_bits), + }) catch unreachable, + } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); + } + + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + const dst_reg = dst_mcv.getReg().?.to128(); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + if (dst_bits == 16) { + assert(self.hasFeature(.f16c)); + switch (src_bits) { + 32 => { + const mat_src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + mat_src_reg.to128(), + bits.RoundMode.imm(.{}), + ); + }, + else => unreachable, + } + } else { + assert(src_bits == 64 and dst_bits == 32); + if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( + .{ .v_ss, .cvtsd2 }, + dst_reg, + dst_reg, + try src_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegisterRegister( + .{ .v_ss, .cvtsd2 }, + dst_reg, + dst_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv)).to128(), + ) else if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ ._ss, .cvtsd2 }, + dst_reg, + try src_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._ss, .cvtsd2 }, + dst_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv)).to128(), + ); + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const dst_ty = self.typeOfIndex(inst); + const dst_scalar_ty = dst_ty.scalarType(zcu); + const dst_bits = dst_scalar_ty.floatBits(self.target); + const src_ty = self.typeOf(ty_op.operand); + const src_scalar_ty = src_ty.scalarType(zcu); + const src_bits = src_scalar_ty.floatBits(self.target); + + const result = result: { + if (switch (src_bits) { + 16 => switch (dst_bits) { + 32, 64 => !self.hasFeature(.f16c), + 80, 128 => true, + else => unreachable, + }, + 32 => switch (dst_bits) { + 64 => false, + 80, 128 => true, + else => unreachable, + }, + 64 => switch (dst_bits) { + 80, 128 => true, + else => unreachable, + }, + 80 => switch (dst_bits) { + 128 => true, + else => unreachable, + }, + else => unreachable, + }) { + if (dst_ty.isVector(zcu)) break :result null; + var sym_buf: ["__extend?f?f2".len]u8 = undefined; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(), + .param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "__extend{c}f{c}f2", .{ + floatCompilerRtAbiName(src_bits), + floatCompilerRtAbiName(dst_bits), + }) catch unreachable, + } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); + } + + const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(zcu), 16))); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const vec_len = if (dst_ty.isVector(zcu)) dst_ty.vectorLen(zcu) else 1; + if (src_bits == 16) { + assert(self.hasFeature(.f16c)); + const mat_src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv); + try self.asmRegisterRegister( + .{ .v_ps, .cvtph2 }, + dst_alias, + registerAlias(mat_src_reg, src_abi_size), + ); + switch (dst_bits) { + 32 => {}, + 64 => try self.asmRegisterRegisterRegister( + .{ .v_sd, .cvtss2 }, + dst_alias, + dst_alias, + dst_alias, + ), + else => unreachable, + } + } else { + assert(src_bits == 32 and dst_bits == 64); + if (self.hasFeature(.avx)) switch (vec_len) { + 1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( + .{ .v_sd, .cvtss2 }, + dst_alias, + dst_alias, + try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), + ) else try self.asmRegisterRegisterRegister( + .{ .v_sd, .cvtss2 }, + dst_alias, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), + ), + 2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_pd, .cvtps2 }, + dst_alias, + try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), + ) else try self.asmRegisterRegister( + .{ .v_pd, .cvtps2 }, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), + ), + else => break :result null, + } else if (src_mcv.isBase()) try self.asmRegisterMemory( + switch (vec_len) { + 1 => .{ ._sd, .cvtss2 }, + 2 => .{ ._pd, .cvtps2 }, + else => break :result null, + }, + dst_alias, + try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), + ) else try self.asmRegisterRegister( + switch (vec_len) { + 1 => .{ ._sd, .cvtss2 }, + 2 => .{ ._pd, .cvtps2 }, + else => break :result null, + }, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), + ); + } + break :result dst_mcv; + } orelse return self.fail("TODO implement airFpext from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const src_ty = self.typeOf(ty_op.operand); + const dst_ty = self.typeOfIndex(inst); + + const result = @as(?MCValue, result: { + const src_abi_size: u31 = @intCast(src_ty.abiSize(zcu)); + const dst_abi_size: u31 = @intCast(dst_ty.abiSize(zcu)); + + const src_int_info = src_ty.intInfo(zcu); + const dst_int_info = dst_ty.intInfo(zcu); + const extend = switch (src_int_info.signedness) { + .signed => dst_int_info, + .unsigned => src_int_info, + }.signedness; + + const src_mcv = try self.resolveInst(ty_op.operand); + if (dst_ty.isVector(zcu)) { + const max_abi_size = @max(dst_abi_size, src_abi_size); + const has_avx = self.hasFeature(.avx); + + const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); + const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu); + switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) { + .lt => { + if (max_abi_size > self.vectorSize(.int)) break :result null; + const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) { + else => break :result null, + 1 => switch (src_elem_abi_size) { + else => break :result null, + 2 => switch (dst_int_info.signedness) { + .signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw }, + .unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw }, + }, + }, + 2 => switch (src_elem_abi_size) { + else => break :result null, + 4 => switch (dst_int_info.signedness) { + .signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd }, + .unsigned => if (has_avx) + .{ .vp_w, .ackusd } + else if (self.hasFeature(.sse4_1)) + .{ .p_w, .ackusd } + else + break :result null, + }, + }, + }; + + const dst_mcv: MCValue = if (src_mcv.isRegister() and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else if (has_avx and src_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, dst_abi_size); + + if (has_avx) try self.asmRegisterRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + dst_reg, src_abi_size), + dst_alias, + ) else try self.asmRegisterRegister( + mir_tag, + dst_alias, + dst_alias, + ); + break :result dst_mcv; + }, + .eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + break :result src_mcv + else { + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); + break :result dst_mcv; + }, + .gt => if (self.hasFeature(.sse4_1)) { + if (max_abi_size > self.vectorSize(.int)) break :result null; + const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) { + else => break :result null, + 2 => if (has_avx) .vp_w else .p_w, + 4 => if (has_avx) .vp_d else .p_d, + 8 => if (has_avx) .vp_q else .p_q, + }, switch (src_elem_abi_size) { + else => break :result null, + 1 => switch (extend) { + .signed => .movsxb, + .unsigned => .movzxb, + }, + 2 => switch (extend) { + .signed => .movsxw, + .unsigned => .movzxw, + }, + 4 => switch (extend) { + .signed => .movsxd, + .unsigned => .movzxd, + }, + } }; + + const dst_mcv: MCValue = if (src_mcv.isRegister() and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }; + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, dst_abi_size); + + if (src_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + dst_alias, + try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), + ) else try self.asmRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), + ); + break :result dst_mcv; + } else { + const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) { + else => break :result null, + 2 => switch (src_elem_abi_size) { + else => break :result null, + 1 => .{ .p_, .unpcklbw }, + }, + 4 => switch (src_elem_abi_size) { + else => break :result null, + 2 => .{ .p_, .unpcklwd }, + }, + 8 => switch (src_elem_abi_size) { + else => break :result null, + 2 => .{ .p_, .unpckldq }, + }, + }; + + const dst_mcv: MCValue = if (src_mcv.isRegister() and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + const dst_reg = dst_mcv.getReg().?; + + const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); + const ext_alias = registerAlias(ext_reg, src_abi_size); + const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg); + defer self.register_manager.unlockReg(ext_lock); + + try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias); + switch (extend) { + .signed => try self.asmRegisterRegister( + .{ switch (src_elem_abi_size) { + else => unreachable, + 1 => .p_b, + 2 => .p_w, + 4 => .p_d, + }, .cmpgt }, + ext_alias, + registerAlias(dst_reg, src_abi_size), + ), + .unsigned => {}, + } + try self.asmRegisterRegister( + mir_tag, + registerAlias(dst_reg, dst_abi_size), + registerAlias(ext_reg, dst_abi_size), + ); + break :result dst_mcv; + }, + } + @compileError("unreachable"); + } + + const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; + + const src_storage_bits: u16 = switch (src_mcv) { + .register, .register_offset => 64, + .register_pair => 128, + .load_frame => |frame_addr| @intCast(self.getFrameAddrSize(frame_addr) * 8), + else => src_int_info.bits, + }; + + const dst_mcv = if ((if (src_mcv.getReg()) |src_reg| src_reg.isClass(.general_purpose) else src_abi_size > 8) and + dst_int_info.bits <= src_storage_bits and + std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable == + std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(min_ty, dst_mcv, src_mcv, .{}); + break :dst dst_mcv; + }; + + if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister()) + .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) } + else + dst_mcv; + + if (dst_mcv.isRegister()) { + try self.truncateRegister(src_ty, dst_mcv.getReg().?); + break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }; + } + + const src_limbs_len = std.math.divCeil(u31, src_abi_size, 8) catch unreachable; + const dst_limbs_len = @divExact(dst_abi_size, 8); + + const high_mcv: MCValue = if (dst_mcv.isBase()) + dst_mcv.address().offset((src_limbs_len - 1) * 8).deref() + else + .{ .register = dst_mcv.register_pair[1] }; + const high_reg = if (high_mcv.isRegister()) + high_mcv.getReg().? + else + try self.copyToTmpRegister(switch (src_int_info.signedness) { + .signed => .isize, + .unsigned => .usize, + }, high_mcv); + const high_lock = self.register_manager.lockRegAssumeUnused(high_reg); + defer self.register_manager.unlockReg(high_lock); + + const high_bits = src_int_info.bits % 64; + if (high_bits > 0) { + try self.truncateRegister(src_ty, high_reg); + const high_ty: Type = if (dst_int_info.bits >= 64) .usize else dst_ty; + try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{}); + } + + if (dst_limbs_len > src_limbs_len) try self.genInlineMemset( + dst_mcv.address().offset(src_limbs_len * 8), + switch (extend) { + .signed => extend: { + const extend_mcv = MCValue{ .register = high_reg }; + try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, extend_mcv, .u8, .{ .immediate = 63 }); + break :extend extend_mcv; + }, + .unsigned => .{ .immediate = 0 }, + }, + .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 }, + .{}, + ); + + break :result dst_mcv; + }) orelse return self.fail("TODO implement airIntCast from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dst_ty = self.typeOfIndex(inst); + const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + const src_ty = self.typeOf(ty_op.operand); + const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); + + const result = result: { + const src_mcv = try self.resolveInst(ty_op.operand); + const src_lock = + if (src_mcv.getReg()) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = if (src_mcv.isRegister() and src_mcv.getReg().?.isClass(self.regClassForType(dst_ty)) and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else if (dst_abi_size <= 8) + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv) + else if (dst_abi_size <= 16 and !dst_ty.isVector(zcu)) dst: { + const dst_regs = + try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp); + const dst_mcv: MCValue = .{ .register_pair = dst_regs }; + const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); + defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); + + try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); + break :dst dst_mcv; + } else dst: { + const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true); + try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); + break :dst dst_mcv; + }; + + if (dst_ty.zigTypeTag(zcu) == .vector) { + assert(src_ty.zigTypeTag(zcu) == .vector and dst_ty.vectorLen(zcu) == src_ty.vectorLen(zcu)); + const dst_elem_ty = dst_ty.childType(zcu); + const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(zcu)); + const src_elem_ty = src_ty.childType(zcu); + const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(zcu)); + + const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) { + 1 => switch (src_elem_abi_size) { + 2 => switch (dst_ty.vectorLen(zcu)) { + 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw }, + 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null, + else => null, + }, + else => null, + }, + 2 => switch (src_elem_abi_size) { + 4 => switch (dst_ty.vectorLen(zcu)) { + 1...4 => if (self.hasFeature(.avx)) + .{ .vp_w, .ackusd } + else if (self.hasFeature(.sse4_1)) + .{ .p_w, .ackusd } + else + null, + 5...8 => if (self.hasFeature(.avx2)) .{ .vp_w, .ackusd } else null, + else => null, + }, + else => null, + }, + else => null, + }) orelse return self.fail("TODO implement airTrunc for {f}", .{dst_ty.fmt(pt)}); + + const dst_info = dst_elem_ty.intInfo(zcu); + const src_info = src_elem_ty.intInfo(zcu); + + const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits)); + + const splat_ty = try pt.vectorType(.{ + .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)), + .child = src_elem_ty.ip_index, + }); + const splat_abi_size: u32 = @intCast(splat_ty.abiSize(zcu)); + + const splat_val = try pt.aggregateSplatValue(splat_ty, mask_val); + + const splat_mcv = try self.lowerValue(splat_val); + const splat_addr_mcv: MCValue = switch (splat_mcv) { + .memory, .indirect, .load_frame => splat_mcv.address(), + else => .{ .register = try self.copyToTmpRegister(.usize, splat_mcv.address()) }, + }; + + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, src_abi_size); + if (self.hasFeature(.avx)) { + try self.asmRegisterRegisterMemory( + .{ .vp_, .@"and" }, + dst_alias, + dst_alias, + try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }), + ); + if (src_abi_size > 16) { + const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); + const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); + defer self.register_manager.unlockReg(temp_lock); + + try self.asmRegisterRegisterImmediate( + .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract }, + registerAlias(temp_reg, dst_abi_size), + dst_alias, + .u(1), + ); + try self.asmRegisterRegisterRegister( + mir_tag, + registerAlias(dst_reg, dst_abi_size), + registerAlias(dst_reg, dst_abi_size), + registerAlias(temp_reg, dst_abi_size), + ); + } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias); + } else { + try self.asmRegisterMemory( + .{ .p_, .@"and" }, + dst_alias, + try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }), + ); + try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias); + } + break :result dst_mcv; + } + + // when truncating a `u16` to `u5`, for example, those top 3 bits in the result + // have to be removed. this only happens if the dst if not a power-of-two size. + if (dst_abi_size <= 8) { + if (self.regExtraBits(dst_ty) > 0) { + try self.truncateRegister(dst_ty, dst_mcv.register.to64()); + } + } else if (dst_abi_size <= 16) { + const dst_info = dst_ty.intInfo(zcu); + const high_ty = try pt.intType(dst_info.signedness, dst_info.bits - 64); + if (self.regExtraBits(high_ty) > 0) { + try self.truncateRegister(high_ty, dst_mcv.register_pair[1].to64()); + } + } + + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + + const slice_ty = self.typeOfIndex(inst); + const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu)); + + const ptr_ty = self.typeOf(bin_op.lhs); + try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{}); + + const len_ty = self.typeOf(bin_op.rhs); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(ptr_ty.abiSize(zcu)), + len_ty, + .{ .air_ref = bin_op.rhs }, + .{}, + ); + + const result = MCValue{ .load_frame = .{ .index = frame_index } }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand); + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); + + const dst_ty = self.typeOfIndex(inst); + if (dst_ty.isAbiInt(zcu)) { + const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + const bit_size: u32 = @intCast(dst_ty.bitSize(zcu)); + if (abi_size * 8 > bit_size) { + const dst_lock = switch (dst_mcv) { + .register => |dst_reg| self.register_manager.lockRegAssumeUnused(dst_reg), + else => null, + }; + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + if (dst_mcv.isRegister()) { + try self.truncateRegister(dst_ty, dst_mcv.getReg().?); + } else { + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1)); + const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref(); + try self.genSetReg(tmp_reg, hi_ty, hi_mcv, .{}); + try self.truncateRegister(dst_ty, tmp_reg); + try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg }, .{}); + } + } + } + return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); + return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 { + const pt = self.pt; + const zcu = pt.zcu; + const air_tag = self.air.instructions.items(.tag); + const air_data = self.air.instructions.items(.data); + + const dst_ty = self.typeOf(dst_air); + const dst_info = dst_ty.intInfo(zcu); + if (dst_air.toIndex()) |inst| { + switch (air_tag[@intFromEnum(inst)]) { + .intcast => { + const src_ty = self.typeOf(air_data[@intFromEnum(inst)].ty_op.operand); + const src_info = src_ty.intInfo(zcu); + return @min(switch (src_info.signedness) { + .signed => switch (dst_info.signedness) { + .signed => src_info.bits, + .unsigned => src_info.bits - 1, + }, + .unsigned => switch (dst_info.signedness) { + .signed => src_info.bits + 1, + .unsigned => src_info.bits, + }, + }, dst_info.bits); + }, + else => {}, + } + } else if (dst_air.toInterned()) |ip_index| { + var space: Value.BigIntSpace = undefined; + const src_int = Value.fromInterned(ip_index).toBigInt(&space, zcu); + return @as(u16, @intCast(src_int.bitCountTwosComp())) + + @intFromBool(src_int.positive and dst_info.signedness == .signed); + } + return dst_info.bits; +} + +fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const result = result: { + const dst_ty = self.typeOfIndex(inst); + switch (dst_ty.zigTypeTag(zcu)) { + .float, .vector => break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs), + else => {}, + } + const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + + const dst_info = dst_ty.intInfo(zcu); + const src_ty = try pt.intType(dst_info.signedness, switch (tag) { + else => unreachable, + .mul, .mul_wrap => @max( + self.activeIntBits(bin_op.lhs), + self.activeIntBits(bin_op.rhs), + dst_info.bits / 2, + ), + .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_info.bits, + }); + const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); + + if (dst_abi_size == 16 and src_abi_size == 16) switch (tag) { + else => unreachable, + .mul, .mul_wrap => {}, + .div_trunc, .div_floor, .div_exact, .rem, .mod => { + const signed = dst_ty.isSignedInt(zcu); + var sym_buf: ["__udiv?i3".len]u8 = undefined; + const signed_div_floor_state: struct { + frame_index: FrameIndex, + state: State, + reloc: Mir.Inst.Index, + } = if (signed and tag == .div_floor) state: { + const frame_index = try self.allocFrameIndex(.initType(.usize, zcu)); + try self.asmMemoryImmediate( + .{ ._, .mov }, + .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, + .u(0), + ); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const mat_lhs_mcv = switch (lhs_mcv) { + .load_nav, .load_uav, .load_lazy_sym => mat_lhs_mcv: { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address()); + break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => lhs_mcv, + }; + const mat_lhs_lock = switch (mat_lhs_mcv) { + .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), + else => null, + }; + defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); + if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .mov }, + tmp_reg, + try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .mov }, + tmp_reg, + mat_lhs_mcv.register_pair[1], + ); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const mat_rhs_mcv = switch (rhs_mcv) { + .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); + break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => rhs_mcv, + }; + const mat_rhs_lock = switch (mat_rhs_mcv) { + .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), + else => null, + }; + defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); + if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .xor }, + tmp_reg, + try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .xor }, + tmp_reg, + mat_rhs_mcv.register_pair[1], + ); + const state = try self.saveState(); + const reloc = try self.asmJccReloc(.ns, undefined); + + break :state .{ .frame_index = frame_index, .state = state, .reloc = reloc }; + } else undefined; + const call_mcv = try self.genCall( + .{ .extern_func = .{ + .return_type = dst_ty.toIntern(), + .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() }, + .sym = std.fmt.bufPrint(&sym_buf, "__{s}{s}{c}i3", .{ + if (signed) "" else "u", + switch (tag) { + .div_trunc, .div_exact => "div", + .div_floor => if (signed) "mod" else "div", + .rem, .mod => "mod", + else => unreachable, + }, + intCompilerRtAbiName(@intCast(dst_ty.bitSize(zcu))), + }) catch unreachable, + } }, + &.{ src_ty, src_ty }, + &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, + .{}, + ); + break :result if (signed) switch (tag) { + .div_floor => { + try self.asmRegisterRegister( + .{ ._, .@"or" }, + call_mcv.register_pair[0], + call_mcv.register_pair[1], + ); + try self.asmSetccMemory(.nz, .{ + .base = .{ .frame = signed_div_floor_state.frame_index }, + .mod = .{ .rm = .{ .size = .byte } }, + }); + try self.restoreState(signed_div_floor_state.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + self.performReloc(signed_div_floor_state.reloc); + const dst_mcv = try self.genCall( + .{ .extern_func = .{ + .return_type = dst_ty.toIntern(), + .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() }, + .sym = std.fmt.bufPrint(&sym_buf, "__div{c}i3", .{ + intCompilerRtAbiName(@intCast(dst_ty.bitSize(zcu))), + }) catch unreachable, + } }, + &.{ src_ty, src_ty }, + &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, + .{}, + ); + try self.asmRegisterMemory( + .{ ._, .sub }, + dst_mcv.register_pair[0], + .{ + .base = .{ .frame = signed_div_floor_state.frame_index }, + .mod = .{ .rm = .{ .size = .qword } }, + }, + ); + try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0)); + try self.freeValue( + .{ .load_frame = .{ .index = signed_div_floor_state.frame_index } }, + ); + break :result dst_mcv; + }, + .mod => { + const dst_regs = call_mcv.register_pair; + const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); + defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); + + const tmp_regs = + try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); + const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); + defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const mat_rhs_mcv = switch (rhs_mcv) { + .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); + break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => rhs_mcv, + }; + const mat_rhs_lock = switch (mat_rhs_mcv) { + .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), + else => null, + }; + defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); + + for (tmp_regs, dst_regs) |tmp_reg, dst_reg| + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg); + if (mat_rhs_mcv.isBase()) { + try self.asmRegisterMemory( + .{ ._, .add }, + tmp_regs[0], + try mat_rhs_mcv.mem(self, .{ .size = .qword }), + ); + try self.asmRegisterMemory( + .{ ._, .adc }, + tmp_regs[1], + try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ); + } else for ( + [_]Mir.Inst.Tag{ .add, .adc }, + tmp_regs, + mat_rhs_mcv.register_pair, + ) |op, tmp_reg, rhs_reg| + try self.asmRegisterRegister(.{ ._, op }, tmp_reg, rhs_reg); + try self.asmRegisterRegister(.{ ._, .@"test" }, dst_regs[1], dst_regs[1]); + for (dst_regs, tmp_regs) |dst_reg, tmp_reg| + try self.asmCmovccRegisterRegister(.s, dst_reg, tmp_reg); + break :result call_mcv; + }, + else => call_mcv, + } else call_mcv; + }, + }; + + try self.spillEflagsIfOccupied(); + try self.spillRegisters(&.{ .rax, .rcx, .rdx }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const rhs_mcv = try self.resolveInst(bin_op.rhs); + break :result try self.genMulDivBinOp(tag, inst, dst_ty, src_ty, lhs_mcv, rhs_mcv); + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ty = self.typeOf(bin_op.lhs); + if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail( + "TODO implement airAddSat for {f}", + .{ty.fmt(pt)}, + ); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) + lhs_mcv + else + try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv); + const dst_reg = dst_mcv.register; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const rhs_lock = switch (rhs_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const limit_mcv = MCValue{ .register = limit_reg }; + const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg); + defer self.register_manager.unlockReg(limit_lock); + + const reg_bits = self.regBitSize(ty); + const reg_extra_bits = self.regExtraBits(ty); + const cc: Condition = if (ty.isSignedInt(zcu)) cc: { + if (reg_extra_bits > 0) { + try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); + } + try self.genSetReg(limit_reg, ty, dst_mcv, .{}); + try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 }); + try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{ + .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1, + }); + if (reg_extra_bits > 0) { + const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv); + const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg }; + const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg); + defer self.register_manager.unlockReg(shifted_rhs_lock); + + try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits }); + try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv); + } else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv); + break :cc .o; + } else cc: { + try self.genSetReg(limit_reg, ty, .{ + .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)), + }, .{}); + + try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv); + if (reg_extra_bits > 0) { + try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, limit_mcv); + break :cc .a; + } + break :cc .c; + }; + + const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); + try self.asmCmovccRegisterRegister( + cc, + registerAlias(dst_reg, cmov_abi_size), + registerAlias(limit_reg, cmov_abi_size), + ); + + if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) + try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); + + return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ty = self.typeOf(bin_op.lhs); + if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail( + "TODO implement airSubSat for {f}", + .{ty.fmt(pt)}, + ); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) + lhs_mcv + else + try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv); + const dst_reg = dst_mcv.register; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const rhs_lock = switch (rhs_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const limit_mcv = MCValue{ .register = limit_reg }; + const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg); + defer self.register_manager.unlockReg(limit_lock); + + const reg_bits = self.regBitSize(ty); + const reg_extra_bits = self.regExtraBits(ty); + const cc: Condition = if (ty.isSignedInt(zcu)) cc: { + if (reg_extra_bits > 0) { + try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); + } + try self.genSetReg(limit_reg, ty, dst_mcv, .{}); + try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 }); + try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{ + .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1, + }); + if (reg_extra_bits > 0) { + const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv); + const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg }; + const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg); + defer self.register_manager.unlockReg(shifted_rhs_lock); + + try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits }); + try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv); + } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv); + break :cc .o; + } else cc: { + try self.genSetReg(limit_reg, ty, .{ .immediate = 0 }, .{}); + try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv); + break :cc .c; + }; + + const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); + try self.asmCmovccRegisterRegister( + cc, + registerAlias(dst_reg, cmov_abi_size), + registerAlias(limit_reg, cmov_abi_size), + ); + + if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) + try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits }); + + return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ty = self.typeOf(bin_op.lhs); + + const result = result: { + if (ty.toIntern() == .i128_type) { + const ptr_c_int = try pt.singleMutPtrType(.c_int); + const overflow = try self.allocTempRegOrMem(.c_int, false); + + const dst_mcv = try self.genCall(.{ .extern_func = .{ + .return_type = .i128_type, + .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() }, + .sym = "__muloti4", + } }, &.{ .i128, .i128, ptr_c_int }, &.{ + .{ .air_ref = bin_op.lhs }, + .{ .air_ref = bin_op.rhs }, + overflow.address(), + }, .{}); + const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair); + defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const mat_lhs_mcv = switch (lhs_mcv) { + .load_nav, .load_uav, .load_lazy_sym => mat_lhs_mcv: { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address()); + break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => lhs_mcv, + }; + const mat_lhs_lock = switch (mat_lhs_mcv) { + .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), + else => null, + }; + defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); + if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .mov }, + tmp_reg, + try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .mov }, + tmp_reg, + mat_lhs_mcv.register_pair[1], + ); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const mat_rhs_mcv = switch (rhs_mcv) { + .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); + break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => rhs_mcv, + }; + const mat_rhs_lock = switch (mat_rhs_mcv) { + .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg), + else => null, + }; + defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); + if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .xor }, + tmp_reg, + try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .xor }, + tmp_reg, + mat_rhs_mcv.register_pair[1], + ); + + try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63)); + try self.asmRegister(.{ ._, .not }, tmp_reg); + try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0)); + try self.freeValue(overflow); + try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg); + try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63)); + try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg); + break :result dst_mcv; + } + + if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail( + "TODO implement airMulSat for {f}", + .{ty.fmt(pt)}, + ); + + try self.spillRegisters(&.{ .rax, .rcx, .rdx }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const lhs_lock = switch (lhs_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const rhs_lock = switch (rhs_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const limit_mcv = MCValue{ .register = limit_reg }; + const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg); + defer self.register_manager.unlockReg(limit_lock); + + const reg_bits = self.regBitSize(ty); + const cc: Condition = if (ty.isSignedInt(zcu)) cc: { + try self.genSetReg(limit_reg, ty, lhs_mcv, .{}); + try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv); + try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 }); + try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{ + .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1, + }); + break :cc .o; + } else cc: { + try self.genSetReg(limit_reg, ty, .{ + .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits), + }, .{}); + break :cc .c; + }; + + const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv); + const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); + try self.asmCmovccRegisterRegister( + cc, + registerAlias(dst_mcv.register, cmov_abi_size), + registerAlias(limit_reg, cmov_abi_size), + ); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const result: MCValue = result: { + const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; + const ty = self.typeOf(bin_op.lhs); + switch (ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO implement add/sub with overflow for Vector type", .{}), + .int => { + try self.spillEflagsIfOccupied(); + try self.spillRegisters(&.{ .rcx, .rdi, .rsi }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rcx, .rdi, .rsi }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const partial_mcv = try self.genBinOp(null, switch (tag) { + .add_with_overflow => .add, + .sub_with_overflow => .sub, + else => unreachable, + }, bin_op.lhs, bin_op.rhs); + const int_info = ty.intInfo(zcu); + const cc: Condition = switch (int_info.signedness) { + .unsigned => .c, + .signed => .o, + }; + + const tuple_ty = self.typeOfIndex(inst); + if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) { + switch (partial_mcv) { + .register => |reg| { + self.eflags_inst = inst; + break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; + }, + else => {}, + } + + const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(1, zcu)), + .u1, + .{ .eflags = cc }, + .{}, + ); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(0, zcu)), + ty, + partial_mcv, + .{}, + ); + break :result .{ .load_frame = .{ .index = frame_index } }; + } + + const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); + try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + else => unreachable, + } + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const result: MCValue = result: { + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + switch (lhs_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO implement shl with overflow for Vector type", .{}), + .int => { + try self.spillEflagsIfOccupied(); + try self.spillRegisters(&.{ .rcx, .rdi, .rsi }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rcx, .rdi, .rsi }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const int_info = lhs_ty.intInfo(zcu); + + const partial_mcv = try self.genShiftBinOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty); + const partial_lock = switch (partial_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (partial_lock) |lock| self.register_manager.unlockReg(lock); + + const tmp_mcv = try self.genShiftBinOp(.shr, null, partial_mcv, rhs, lhs_ty, rhs_ty); + const tmp_lock = switch (tmp_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs); + const cc = Condition.ne; + + const tuple_ty = self.typeOfIndex(inst); + if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) { + switch (partial_mcv) { + .register => |reg| { + self.eflags_inst = inst; + break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; + }, + else => {}, + } + + const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(1, zcu)), + tuple_ty.fieldType(1, zcu), + .{ .eflags = cc }, + .{}, + ); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(0, zcu)), + tuple_ty.fieldType(0, zcu), + partial_mcv, + .{}, + ); + break :result .{ .load_frame = .{ .index = frame_index } }; + } + + const frame_index = + try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); + try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + else => unreachable, + } + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn genSetFrameTruncatedOverflowCompare( + self: *CodeGen, + tuple_ty: Type, + frame_index: FrameIndex, + src_mcv: MCValue, + overflow_cc: ?Condition, +) !void { + const pt = self.pt; + const zcu = pt.zcu; + const src_lock = switch (src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const ty = tuple_ty.fieldType(0, zcu); + const ty_size = ty.abiSize(zcu); + const int_info = ty.intInfo(zcu); + + const hi_bits = (int_info.bits - 1) % 64 + 1; + const hi_ty = try pt.intType(int_info.signedness, hi_bits); + + const limb_bits: u16 = @intCast(if (int_info.bits <= 64) self.regBitSize(ty) else 64); + const limb_ty = try pt.intType(int_info.signedness, limb_bits); + + const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits); + + const temp_regs = + try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp); + const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); + defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); + + const overflow_reg = temp_regs[0]; + if (overflow_cc) |cc| try self.asmSetccRegister(cc, overflow_reg.to8()); + + const scratch_reg = temp_regs[1]; + const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8; + const hi_limb_mcv = if (hi_limb_off > 0) + src_mcv.address().offset(int_info.bits / 64 * 8).deref() + else + src_mcv; + try self.genSetReg(scratch_reg, limb_ty, hi_limb_mcv, .{}); + try self.truncateRegister(hi_ty, scratch_reg); + try self.genBinOpMir(.{ ._, .cmp }, limb_ty, .{ .register = scratch_reg }, hi_limb_mcv); + + const eq_reg = temp_regs[2]; + if (overflow_cc) |_| { + try self.asmSetccRegister(.ne, eq_reg.to8()); + try self.genBinOpMir(.{ ._, .@"or" }, .u8, .{ .register = overflow_reg }, .{ .register = eq_reg }); + } + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(1, zcu)), + tuple_ty.fieldType(1, zcu), + if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne }, + .{}, + ); + + const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu)); + if (hi_limb_off > 0) try self.genSetMem( + .{ .frame = frame_index }, + payload_off, + rest_ty, + src_mcv, + .{}, + ); + try self.genSetMem( + .{ .frame = frame_index }, + payload_off + hi_limb_off, + limb_ty, + .{ .register = scratch_reg }, + .{}, + ); + var ext_off: i32 = hi_limb_off + 8; + if (ext_off < ty_size) { + switch (int_info.signedness) { + .signed => try self.asmRegisterImmediate(.{ ._r, .sa }, scratch_reg.to64(), .s(63)), + .unsigned => try self.asmRegisterRegister(.{ ._, .xor }, scratch_reg.to32(), scratch_reg.to32()), + } + while (ext_off < ty_size) : (ext_off += 8) try self.genSetMem( + .{ .frame = frame_index }, + payload_off + ext_off, + limb_ty, + .{ .register = scratch_reg }, + .{}, + ); + } +} + +fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const tuple_ty = self.typeOfIndex(inst); + const dst_ty = self.typeOf(bin_op.lhs); + const result: MCValue = switch (dst_ty.zigTypeTag(zcu)) { + .vector => return self.fail("TODO implement airMulWithOverflow for {f}", .{dst_ty.fmt(pt)}), + .int => result: { + const dst_info = dst_ty.intInfo(zcu); + if (dst_info.bits > 128 and dst_info.signedness == .unsigned) { + const slow_inc = self.hasFeature(.slow_incdec); + const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; + + try self.spillRegisters(&.{ .rax, .rcx, .rdx }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.genInlineMemset( + dst_mcv.address(), + .{ .immediate = 0 }, + .{ .immediate = tuple_ty.abiSize(zcu) }, + .{}, + ); + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const rhs_mcv = try self.resolveInst(bin_op.rhs); + + const temp_regs = + try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); + const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs); + defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); + + try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32()); + + const outer_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[1].to64(), .{ + .base = .{ .frame = rhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[0].to64(), + .scale = .@"8", + .disp = rhs_mcv.load_frame.off, + } }, + }); + try self.asmRegisterRegister(.{ ._, .@"test" }, temp_regs[1].to64(), temp_regs[1].to64()); + const skip_inner = try self.asmJccReloc(.z, undefined); + + try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[2].to32(), temp_regs[2].to32()); + try self.asmRegisterRegister(.{ ._, .mov }, temp_regs[3].to32(), temp_regs[0].to32()); + try self.asmRegisterRegister(.{ ._, .xor }, .ecx, .ecx); + try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx); + + const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1)); + try self.asmMemoryRegister(.{ ._, .adc }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[3].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off + + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), + } }, + }, .rdx); + try self.asmSetccRegister(.c, .cl); + + try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[2].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off, + } }, + }); + try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64()); + + try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1)); + try self.asmMemoryRegister(.{ ._, .adc }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[3].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off + + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))), + } }, + }, .rax); + try self.asmSetccRegister(.c, .ch); + + if (slow_inc) { + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, temp_regs[2].to32()); + try self.asmRegister(.{ ._c, .in }, temp_regs[3].to32()); + } + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len)); + _ = try self.asmJccReloc(.b, inner_loop); + + try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx); + const overflow = try self.asmJccReloc(.nz, undefined); + const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len)); + const no_overflow = try self.asmJccReloc(.nb, undefined); + if (slow_inc) { + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, temp_regs[2].to32()); + } + try self.asmMemoryImmediate(.{ ._, .cmp }, .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[2].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off - 8, + } }, + }, .u(0)); + _ = try self.asmJccReloc(.z, overflow_loop); + self.performReloc(overflow); + try self.asmMemoryImmediate(.{ ._, .mov }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .byte, + .disp = dst_mcv.load_frame.off + + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), + } }, + }, .u(1)); + self.performReloc(no_overflow); + + self.performReloc(skip_inner); + if (slow_inc) { + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32()); + } + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len)); + _ = try self.asmJccReloc(.b, outer_loop); + + break :result dst_mcv; + } + + const lhs_active_bits = self.activeIntBits(bin_op.lhs); + const rhs_active_bits = self.activeIntBits(bin_op.rhs); + const src_bits = @max(lhs_active_bits, rhs_active_bits, dst_info.bits / 2); + const src_ty = try pt.intType(dst_info.signedness, src_bits); + if (src_bits > 64 and src_bits <= 128 and + dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) { + .signed => { + const ptr_c_int = try pt.singleMutPtrType(.c_int); + const overflow = try self.allocTempRegOrMem(.c_int, false); + const result = try self.genCall(.{ .extern_func = .{ + .return_type = .i128_type, + .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() }, + .sym = "__muloti4", + } }, &.{ .i128, .i128, ptr_c_int }, &.{ + .{ .air_ref = bin_op.lhs }, + .{ .air_ref = bin_op.rhs }, + overflow.address(), + }, .{}); + + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.genSetMem( + .{ .frame = dst_mcv.load_frame.index }, + @intCast(tuple_ty.structFieldOffset(0, zcu)), + tuple_ty.fieldType(0, zcu), + result, + .{}, + ); + try self.asmMemoryImmediate( + .{ ._, .cmp }, + try overflow.mem(self, .{ .size = self.memSize(.c_int) }), + .s(0), + ); + try self.genSetMem( + .{ .frame = dst_mcv.load_frame.index }, + @intCast(tuple_ty.structFieldOffset(1, zcu)), + tuple_ty.fieldType(1, zcu), + .{ .eflags = .ne }, + .{}, + ); + try self.freeValue(overflow); + break :result dst_mcv; + }, + .unsigned => { + try self.spillEflagsIfOccupied(); + try self.spillRegisters(&.{ .rax, .rdx }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const tmp_regs = + try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); + const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs); + defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const mat_lhs_mcv = mat_lhs_mcv: switch (lhs_mcv) { + .register => |lhs_reg| switch (lhs_reg.class()) { + else => lhs_mcv, + .sse => { + const mat_lhs_mcv: MCValue = .{ + .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp), + }; + try self.genCopy(dst_ty, mat_lhs_mcv, lhs_mcv, .{}); + break :mat_lhs_mcv mat_lhs_mcv; + }, + }, + .load_nav, .load_uav, .load_lazy_sym => { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address()); + break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => lhs_mcv, + }; + const mat_lhs_locks: [2]?RegisterLock = switch (mat_lhs_mcv) { + .register_pair => |mat_lhs_regs| self.register_manager.lockRegs(2, mat_lhs_regs), + .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null }, + else => @splat(null), + }; + defer for (mat_lhs_locks) |mat_lhs_lock| if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); + const mat_rhs_mcv = mat_rhs_mcv: switch (rhs_mcv) { + .register => |rhs_reg| switch (rhs_reg.class()) { + else => rhs_mcv, + .sse => { + const mat_rhs_mcv: MCValue = .{ + .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp), + }; + try self.genCopy(dst_ty, mat_rhs_mcv, rhs_mcv, .{}); + break :mat_rhs_mcv mat_rhs_mcv; + }, + }, + .load_nav, .load_uav, .load_lazy_sym => { + // TODO clean this up! + const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address()); + break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } }; + }, + else => rhs_mcv, + }; + const mat_rhs_locks: [2]?RegisterLock = switch (mat_rhs_mcv) { + .register_pair => |mat_rhs_regs| self.register_manager.lockRegs(2, mat_rhs_regs), + .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null }, + else => @splat(null), + }; + defer for (mat_rhs_locks) |mat_rhs_lock| if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); + + if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .mov }, + .rax, + try mat_lhs_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .mov }, + .rax, + mat_lhs_mcv.register_pair[0], + ); + if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .mov }, + tmp_regs[0], + try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .mov }, + tmp_regs[0], + mat_rhs_mcv.register_pair[1], + ); + try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_regs[0], tmp_regs[0]); + try self.asmSetccRegister(.nz, tmp_regs[1].to8()); + try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax); + try self.asmSetccRegister(.o, tmp_regs[2].to8()); + if (mat_rhs_mcv.isBase()) + try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) + else + try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); + try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); + try self.asmSetccRegister(.c, tmp_regs[3].to8()); + try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8()); + if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .mov }, + tmp_regs[0], + try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._, .mov }, + tmp_regs[0], + mat_lhs_mcv.register_pair[1], + ); + try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_regs[0], tmp_regs[0]); + try self.asmSetccRegister(.nz, tmp_regs[3].to8()); + try self.asmRegisterRegister( + .{ ._, .@"and" }, + tmp_regs[1].to8(), + tmp_regs[3].to8(), + ); + try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); + if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( + .{ .i_, .mul }, + tmp_regs[0], + try mat_rhs_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ .i_, .mul }, + tmp_regs[0], + mat_rhs_mcv.register_pair[0], + ); + try self.asmSetccRegister(.o, tmp_regs[2].to8()); + try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); + try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); + try self.asmSetccRegister(.c, tmp_regs[2].to8()); + try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); + + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.genSetMem( + .{ .frame = dst_mcv.load_frame.index }, + @intCast(tuple_ty.structFieldOffset(0, zcu)), + tuple_ty.fieldType(0, zcu), + .{ .register_pair = .{ .rax, .rdx } }, + .{}, + ); + try self.genSetMem( + .{ .frame = dst_mcv.load_frame.index }, + @intCast(tuple_ty.structFieldOffset(1, zcu)), + tuple_ty.fieldType(1, zcu), + .{ .register = tmp_regs[1] }, + .{}, + ); + break :result dst_mcv; + }, + }; + + try self.spillEflagsIfOccupied(); + try self.spillRegisters(&.{ .rax, .rcx, .rdx, .rdi, .rsi }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(5, .{ .rax, .rcx, .rdx, .rdi, .rsi }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const cc: Condition = switch (dst_info.signedness) { + .unsigned => .c, + .signed => .o, + }; + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const extra_bits = if (dst_info.bits <= 64) + self.regExtraBits(dst_ty) + else + dst_info.bits % 64; + const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs); + + switch (partial_mcv) { + .register => |reg| if (extra_bits == 0) { + self.eflags_inst = inst; + break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; + } else { + const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); + try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + else => { + // For now, this is the only supported multiply that doesn't fit in a register. + if (dst_info.bits > 128 or src_bits != 64) + return self.fail("TODO implement airWithOverflow from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + + const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu)); + if (dst_info.bits >= lhs_active_bits + rhs_active_bits) { + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(0, zcu)), + tuple_ty.fieldType(0, zcu), + partial_mcv, + .{}, + ); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(tuple_ty.structFieldOffset(1, zcu)), + tuple_ty.fieldType(1, zcu), + .{ .immediate = 0 }, // cc being set is impossible + .{}, + ); + } else try self.genSetFrameTruncatedOverflowCompare( + tuple_ty, + frame_index, + partial_mcv, + null, + ); + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + } + }, + else => unreachable, + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +/// Generates signed or unsigned integer multiplication/division. +/// Clobbers .rax and .rdx registers. +/// Quotient is saved in .rax and remainder in .rdx. +fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void { + const pt = self.pt; + const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); + const bit_size: u32 = @intCast(self.regBitSize(ty)); + if (abi_size > 8) { + return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{}); + } + + try self.genSetReg(.rax, ty, lhs, .{}); + switch (tag[1]) { + else => unreachable, + .mul => {}, + .div => switch (tag[0]) { + ._ => { + const hi_reg: Register = + switch (bit_size) { + 8 => .ah, + 16, 32, 64 => .edx, + else => unreachable, + }; + try self.asmRegisterRegister(.{ ._, .xor }, hi_reg, hi_reg); + }, + .i_ => try self.asmOpOnly(.{ ._, switch (bit_size) { + 8 => .cbw, + 16 => .cwd, + 32 => .cdq, + 64 => .cqo, + else => unreachable, + } }), + else => unreachable, + }, + } + + const mat_rhs: MCValue = switch (rhs) { + .register, .indirect, .load_frame => rhs, + else => .{ .register = try self.copyToTmpRegister(ty, rhs) }, + }; + switch (mat_rhs) { + .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)), + .memory, .indirect, .load_frame => try self.asmMemory( + tag, + try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }), + ), + else => unreachable, + } + if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah); +} + +/// Always returns a register. +/// Clobbers .rax and .rdx registers. +fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const int_info = ty.intInfo(zcu); + const dividend = switch (lhs) { + .register => |reg| reg, + else => try self.copyToTmpRegister(ty, lhs), + }; + const dividend_lock = self.register_manager.lockReg(dividend); + defer if (dividend_lock) |lock| self.register_manager.unlockReg(lock); + + const divisor = switch (rhs) { + .register => |reg| reg, + else => try self.copyToTmpRegister(ty, rhs), + }; + const divisor_lock = self.register_manager.lockReg(divisor); + defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genIntMulDivOpMir( + switch (int_info.signedness) { + .signed => .{ .i_, .div }, + .unsigned => .{ ._, .div }, + }, + ty, + .{ .register = dividend }, + .{ .register = divisor }, + ); + + try self.asmRegisterRegister( + .{ ._, .xor }, + registerAlias(divisor, abi_size), + registerAlias(dividend, abi_size), + ); + try self.asmRegisterImmediate( + .{ ._r, .sa }, + registerAlias(divisor, abi_size), + .u(int_info.bits - 1), + ); + try self.asmRegisterRegister( + .{ ._, .@"test" }, + registerAlias(.rdx, abi_size), + registerAlias(.rdx, abi_size), + ); + try self.asmCmovccRegisterRegister( + .z, + registerAlias(divisor, @max(abi_size, 2)), + registerAlias(.rdx, @max(abi_size, 2)), + ); + try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax }); + return MCValue{ .register = divisor }; +} + +fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const air_tags = self.air.instructions.items(.tag); + const tag = air_tags[@intFromEnum(inst)]; + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + const result: MCValue = result: { + switch (lhs_ty.zigTypeTag(zcu)) { + .int => { + try self.spillRegisters(&.{.rcx}); + try self.register_manager.getKnownReg(.rcx, null); + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const rhs_mcv = try self.resolveInst(bin_op.rhs); + + const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty); + switch (tag) { + .shr, .shr_exact, .shl_exact => {}, + .shl => switch (dst_mcv) { + .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg), + .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]), + .load_frame => |frame_addr| { + const tmp_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu)); + const tmp_ty: Type = if (lhs_bits > 64) .usize else lhs_ty; + const off = frame_addr.off + (lhs_bits - 1) / 64 * 8; + try self.genSetReg( + tmp_reg, + tmp_ty, + .{ .load_frame = .{ .index = frame_addr.index, .off = off } }, + .{}, + ); + try self.truncateRegister(lhs_ty, tmp_reg); + try self.genSetMem( + .{ .frame = frame_addr.index }, + off, + tmp_ty, + .{ .register = tmp_reg }, + .{}, + ); + }, + else => {}, + }, + else => unreachable, + } + break :result dst_mcv; + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(zcu).intInfo(zcu).bits) { + else => null, + 16 => switch (lhs_ty.vectorLen(zcu)) { + else => null, + 1...8 => switch (tag) { + else => unreachable, + .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_w, .sra } + else + .{ .p_w, .sra }, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_w, .srl } + else + .{ .p_w, .srl }, + }, + .shl, .shl_exact => if (self.hasFeature(.avx)) + .{ .vp_w, .sll } + else + .{ .p_w, .sll }, + }, + 9...16 => switch (tag) { + else => unreachable, + .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null, + .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null, + }, + .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null, + }, + }, + 32 => switch (lhs_ty.vectorLen(zcu)) { + else => null, + 1...4 => switch (tag) { + else => unreachable, + .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_d, .sra } + else + .{ .p_d, .sra }, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_d, .srl } + else + .{ .p_d, .srl }, + }, + .shl, .shl_exact => if (self.hasFeature(.avx)) + .{ .vp_d, .sll } + else + .{ .p_d, .sll }, + }, + 5...8 => switch (tag) { + else => unreachable, + .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null, + .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null, + }, + .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null, + }, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + else => null, + 1...2 => switch (tag) { + else => unreachable, + .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_q, .sra } + else + .{ .p_q, .sra }, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_q, .srl } + else + .{ .p_q, .srl }, + }, + .shl, .shl_exact => if (self.hasFeature(.avx)) + .{ .vp_q, .sll } + else + .{ .p_q, .sll }, + }, + 3...4 => switch (tag) { + else => unreachable, + .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null, + .unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null, + }, + .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null, + }, + }, + })) |mir_tag| if (try self.air.value(bin_op.rhs, pt)) |rhs_val| { + switch (zcu.intern_pool.indexToKey(rhs_val.toIntern())) { + .aggregate => |rhs_aggregate| switch (rhs_aggregate.storage) { + .repeated_elem => |rhs_elem| { + const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) + .{lhs_mcv.getReg().?} ** 2 + else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{ + try self.register_manager.allocReg(inst, abi.RegisterClass.sse), + lhs_mcv.getReg().?, + } else .{(try self.copyToRegisterWithInstTracking( + inst, + lhs_ty, + lhs_mcv, + )).register} ** 2; + const reg_locks = + self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg }); + defer for (reg_locks) |reg_lock| if (reg_lock) |lock| + self.register_manager.unlockReg(lock); + + const shift_imm: Immediate = + .u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu))); + if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate( + mir_tag, + registerAlias(dst_reg, abi_size), + registerAlias(lhs_reg, abi_size), + shift_imm, + ) else { + assert(dst_reg.id() == lhs_reg.id()); + try self.asmRegisterImmediate( + mir_tag, + registerAlias(dst_reg, abi_size), + shift_imm, + ); + } + break :result .{ .register = dst_reg }; + }, + else => {}, + }, + else => {}, + } + } else if (bin_op.rhs.toIndex()) |rhs_inst| switch (air_tags[@intFromEnum(rhs_inst)]) { + .splat => { + const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) + .{lhs_mcv.getReg().?} ** 2 + else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{ + try self.register_manager.allocReg(inst, abi.RegisterClass.sse), + lhs_mcv.getReg().?, + } else .{(try self.copyToRegisterWithInstTracking( + inst, + lhs_ty, + lhs_mcv, + )).register} ** 2; + const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg }); + defer for (reg_locks) |reg_lock| if (reg_lock) |lock| + self.register_manager.unlockReg(lock); + + const shift_reg = + try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs }); + const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg); + defer self.register_manager.unlockReg(shift_lock); + + const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type }); + const mask_mcv = try self.lowerValue(try pt.aggregateValue( + mask_ty, + &([1]InternPool.Index{ + (try rhs_ty.childType(zcu).maxIntScalar(pt, .u8)).toIntern(), + } ++ [1]InternPool.Index{.zero_u8} ** 15), + )); + const mask_addr_reg = try self.copyToTmpRegister(.usize, mask_mcv.address()); + const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg); + defer self.register_manager.unlockReg(mask_addr_lock); + + if (self.hasFeature(.avx)) { + try self.asmRegisterRegisterMemory( + .{ .vp_, .@"and" }, + shift_reg.to128(), + shift_reg.to128(), + .{ + .base = .{ .reg = mask_addr_reg }, + .mod = .{ .rm = .{ .size = .xword } }, + }, + ); + try self.asmRegisterRegisterRegister( + mir_tag, + registerAlias(dst_reg, abi_size), + registerAlias(lhs_reg, abi_size), + shift_reg.to128(), + ); + } else { + try self.asmRegisterMemory( + .{ .p_, .@"and" }, + shift_reg.to128(), + .{ + .base = .{ .reg = mask_addr_reg }, + .mod = .{ .rm = .{ .size = .xword } }, + }, + ); + assert(dst_reg.id() == lhs_reg.id()); + try self.asmRegisterRegister( + mir_tag, + registerAlias(dst_reg, abi_size), + shift_reg.to128(), + ); + } + break :result .{ .register = dst_reg }; + }, + else => {}, + }, + else => {}, + }, + else => {}, + } + return self.fail("TODO implement airShlShrBinOp for {f}", .{lhs_ty.fmt(pt)}); + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs_ty = self.typeOf(bin_op.lhs); + const rhs_ty = self.typeOf(bin_op.rhs); + + const result: MCValue = result: { + switch (lhs_ty.zigTypeTag(zcu)) { + .int => { + const lhs_bits = lhs_ty.bitSize(zcu); + const rhs_bits = rhs_ty.bitSize(zcu); + if (!(lhs_bits <= 32 and rhs_bits <= 5) and !(lhs_bits > 32 and lhs_bits <= 64 and rhs_bits <= 6) and !(rhs_bits <= std.math.log2(lhs_bits))) { + return self.fail("TODO implement shl_sat for {} with lhs bits {}, rhs bits {}", .{ self.target.cpu.arch, lhs_bits, rhs_bits }); + } + + // clobberred by genShiftBinOp + try self.spillRegisters(&.{.rcx}); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + var lhs_temp1 = try self.tempInit(lhs_ty, lhs_mcv); + const rhs_mcv = try self.resolveInst(bin_op.rhs); + + const lhs_lock = switch (lhs_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + // shift left + const dst_mcv = try self.genShiftBinOp(.shl, null, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty); + switch (dst_mcv) { + .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg), + .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]), + .load_frame => |frame_addr| { + const tmp_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const lhs_bits_u31: u31 = @intCast(lhs_bits); + const tmp_ty: Type = if (lhs_bits_u31 > 64) .usize else lhs_ty; + const off = frame_addr.off + (lhs_bits_u31 - 1) / 64 * 8; + try self.genSetReg( + tmp_reg, + tmp_ty, + .{ .load_frame = .{ .index = frame_addr.index, .off = off } }, + .{}, + ); + try self.truncateRegister(lhs_ty, tmp_reg); + try self.genSetMem( + .{ .frame = frame_addr.index }, + off, + tmp_ty, + .{ .register = tmp_reg }, + .{}, + ); + }, + else => {}, + } + const dst_lock = switch (dst_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + // shift right + const tmp_mcv = try self.genShiftBinOp(.shr, null, dst_mcv, rhs_mcv, lhs_ty, rhs_ty); + var tmp_temp = try self.tempInit(lhs_ty, tmp_mcv); + + // check if overflow happens + const cc_temp = lhs_temp1.cmpInts(.neq, &tmp_temp, self) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try lhs_temp1.die(self); + try tmp_temp.die(self); + const overflow_reloc = try self.genCondBrMir(lhs_ty, cc_temp.tracking(self).short); + try cc_temp.die(self); + + // if overflow, + // for unsigned integers, the saturating result is just its max + // for signed integers, + // if lhs is positive, the result is its max + // if lhs is negative, it is min + switch (lhs_ty.intInfo(zcu).signedness) { + .unsigned => { + const bound_mcv = try self.lowerValue(try lhs_ty.maxIntScalar(self.pt, lhs_ty)); + try self.genCopy(lhs_ty, dst_mcv, bound_mcv, .{}); + }, + .signed => { + // check the sign of lhs + // TODO: optimize this. + // we only need the highest bit so shifting the highest part of lhs_mcv + // is enough to check the signedness. other parts can be skipped here. + var lhs_temp2 = try self.tempInit(lhs_ty, lhs_mcv); + var zero_temp = try self.tempInit(lhs_ty, try self.lowerValue(try self.pt.intValue(lhs_ty, 0))); + const sign_cc_temp = lhs_temp2.cmpInts(.lt, &zero_temp, self) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try lhs_temp2.die(self); + try zero_temp.die(self); + const sign_reloc_condbr = try self.genCondBrMir(lhs_ty, sign_cc_temp.tracking(self).short); + try sign_cc_temp.die(self); + + // if it is negative + const min_mcv = try self.lowerValue(try lhs_ty.minIntScalar(self.pt, lhs_ty)); + try self.genCopy(lhs_ty, dst_mcv, min_mcv, .{}); + const sign_reloc_br = try self.asmJmpReloc(undefined); + self.performReloc(sign_reloc_condbr); + + // if it is positive + const max_mcv = try self.lowerValue(try lhs_ty.maxIntScalar(self.pt, lhs_ty)); + try self.genCopy(lhs_ty, dst_mcv, max_mcv, .{}); + self.performReloc(sign_reloc_br); + }, + } + + self.performReloc(overflow_reloc); + break :result dst_mcv; + }, + else => { + return self.fail("TODO implement shl_sat for {} op type {}", .{ self.target.cpu.arch, lhs_ty.zigTypeTag(zcu) }); + }, + } + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = result: { + const pl_ty = self.typeOfIndex(inst); + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const opt_mcv = try self.resolveInst(ty_op.operand); + if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) { + const pl_mcv: MCValue = switch (opt_mcv) { + .register_overflow => |ro| pl: { + self.eflags_inst = null; // actually stop tracking the overflow part + break :pl .{ .register = ro.reg }; + }, + else => opt_mcv, + }; + switch (pl_mcv) { + .register => |pl_reg| try self.truncateRegister(pl_ty, pl_reg), + else => {}, + } + break :result pl_mcv; + } + + const pl_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) { + else => opt_mcv, + .register_overflow => |ro| .{ .register = ro.reg }, + }, .{}); + break :result pl_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dst_ty = self.typeOfIndex(inst); + const opt_mcv = try self.resolveInst(ty_op.operand); + + const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) + opt_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv); + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = result: { + const dst_ty = self.typeOfIndex(inst); + const src_ty = self.typeOf(ty_op.operand); + const opt_ty = src_ty.childType(zcu); + const src_mcv = try self.resolveInst(ty_op.operand); + + if (opt_ty.optionalReprIsPayload(zcu)) { + break :result if (self.liveness.isUnused(inst)) + .unreach + else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + } + + const dst_mcv: MCValue = if (src_mcv.isRegister() and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else if (self.liveness.isUnused(inst)) + .{ .register = try self.copyToTmpRegister(dst_ty, src_mcv) } + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + + const pl_ty = dst_ty.childType(zcu); + const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); + try self.genSetMem( + .{ .reg = dst_mcv.getReg().? }, + pl_abi_size, + .bool, + .{ .immediate = 1 }, + .{}, + ); + break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const err_union_ty = self.typeOf(ty_op.operand); + const err_ty = err_union_ty.errorUnionSet(zcu); + const payload_ty = err_union_ty.errorUnionPayload(zcu); + const operand = try self.resolveInst(ty_op.operand); + + const result: MCValue = result: { + if (err_ty.errorSetIsEmpty(zcu)) { + break :result MCValue{ .immediate = 0 }; + } + + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + break :result try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); + } + + const err_off = codegen.errUnionErrorOffset(payload_ty, zcu); + switch (operand) { + .register => |reg| { + // TODO reuse operand + const eu_lock = self.register_manager.lockReg(reg); + defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); + + const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); + if (err_off > 0) try self.genShiftBinOpMir( + .{ ._r, .sh }, + err_union_ty, + result, + .u8, + .{ .immediate = @as(u6, @intCast(err_off * 8)) }, + ) else try self.truncateRegister(.anyerror, result.register); + break :result result; + }, + .load_frame => |frame_addr| break :result .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + @as(i32, @intCast(err_off)), + } }, + else => return self.fail("TODO implement unwrap_err_err for {f}", .{operand}), + } + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand_ty = self.typeOf(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); + const result = try self.genUnwrapErrUnionPayloadMir(inst, operand_ty, operand); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +// *(E!T) -> E +fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const src_ty = self.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const eu_ty = src_ty.childType(zcu); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); + const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu)); + try self.asmRegisterMemory( + .{ ._, .mov }, + registerAlias(dst_reg, err_abi_size), + .{ + .base = .{ .reg = src_reg }, + .mod = .{ .rm = .{ + .size = .fromSize(err_abi_size), + .disp = err_off, + } }, + }, + ); + + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +// *(E!T) -> *T +fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand_ty = self.typeOf(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); + const result = try self.genUnwrapErrUnionPayloadPtrMir(inst, operand_ty, operand); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = result: { + const src_ty = self.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const eu_ty = src_ty.childType(zcu); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); + const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu)); + try self.asmMemoryImmediate( + .{ ._, .mov }, + .{ + .base = .{ .reg = src_reg }, + .mod = .{ .rm = .{ + .size = .fromSize(err_abi_size), + .disp = err_off, + } }, + }, + .u(0), + ); + + if (self.liveness.isUnused(inst)) break :result .unreach; + + const dst_ty = self.typeOfIndex(inst); + const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_reg + else + try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); + const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + try self.asmRegisterMemory( + .{ ._, .lea }, + registerAlias(dst_reg, dst_abi_size), + .{ + .base = .{ .reg = src_reg }, + .mod = .{ .rm = .{ .disp = pl_off } }, + }, + ); + break :result .{ .register = dst_reg }; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn genUnwrapErrUnionPayloadMir( + self: *CodeGen, + maybe_inst: ?Air.Inst.Index, + err_union_ty: Type, + err_union: MCValue, +) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const payload_ty = err_union_ty.errorUnionPayload(zcu); + + const result: MCValue = result: { + if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu)); + switch (err_union) { + .load_frame => |frame_addr| break :result .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + payload_off, + } }, + .register => |reg| { + // TODO reuse operand + const eu_lock = self.register_manager.lockReg(reg); + defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); + + const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp); + const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null) + try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union) + else + .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) }; + if (payload_off > 0) try self.genShiftBinOpMir( + .{ ._r, .sh }, + err_union_ty, + result_mcv, + .u8, + .{ .immediate = @as(u6, @intCast(payload_off * 8)) }, + ) else try self.truncateRegister(payload_ty, result_mcv.register); + break :result if (payload_in_gp) + result_mcv + else if (maybe_inst) |inst| + try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv) + else + .{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) }; + }, + else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {f}", .{err_union}), + } + }; + + return result; +} + +fn genUnwrapErrUnionPayloadPtrMir( + self: *CodeGen, + maybe_inst: ?Air.Inst.Index, + ptr_ty: Type, + ptr_mcv: MCValue, +) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const err_union_ty = ptr_ty.childType(zcu); + const payload_ty = err_union_ty.errorUnionPayload(zcu); + + const result: MCValue = result: { + const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu); + const result_mcv: MCValue = if (maybe_inst) |inst| + try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv) + else + .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }; + try self.genBinOpMir(.{ ._, .add }, ptr_ty, result_mcv, .{ .immediate = payload_off }); + break :result result_mcv; + }; + + return result; +} + +fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = result: { + const pl_ty = self.typeOf(ty_op.operand); + if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 }; + + const opt_ty = self.typeOfIndex(inst); + const pl_mcv = try self.resolveInst(ty_op.operand); + const same_repr = opt_ty.optionalReprIsPayload(zcu); + if (same_repr and self.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv; + + const pl_lock: ?RegisterLock = switch (pl_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (pl_lock) |lock| self.register_manager.unlockReg(lock); + + const opt_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(pl_ty, opt_mcv, pl_mcv, .{}); + + if (!same_repr) { + const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu)); + switch (opt_mcv) { + else => unreachable, + + .register => |opt_reg| { + try self.truncateRegister(pl_ty, opt_reg); + try self.asmRegisterImmediate( + .{ ._s, .bt }, + opt_reg, + .u(@as(u6, @intCast(pl_abi_size * 8))), + ); + }, + + .load_frame => |frame_addr| try self.asmMemoryImmediate( + .{ ._, .mov }, + .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .byte, + .disp = frame_addr.off + pl_abi_size, + } }, + }, + .u(1), + ), + } + } + break :result opt_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +/// T to E!T +fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const eu_ty = ty_op.ty.toType(); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + const operand = try self.resolveInst(ty_op.operand); + + const result: MCValue = result: { + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 }; + + const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu)); + const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{}); + try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{}); + break :result .{ .load_frame = .{ .index = frame_index } }; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +/// E to E!T +fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const eu_ty = ty_op.ty.toType(); + const pl_ty = eu_ty.errorUnionPayload(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + + const result: MCValue = result: { + if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand); + + const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu)); + const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{}); + const operand = try self.resolveInst(ty_op.operand); + try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{}); + break :result .{ .load_frame = .{ .index = frame_index } }; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = result: { + const src_mcv = try self.resolveInst(ty_op.operand); + const ptr_mcv: MCValue = switch (src_mcv) { + .register_pair => |regs| .{ .register = regs[0] }, + else => src_mcv, + }; + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + switch (src_mcv) { + .register_pair => |regs| try self.freeValue(.{ .register = regs[1] }), + else => {}, + } + break :result ptr_mcv; + } + + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{}); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = result: { + const src_mcv = try self.resolveInst(ty_op.operand); + const len_mcv: MCValue = switch (src_mcv) { + .register_pair => |regs| .{ .register = regs[1] }, + .load_frame => |frame_addr| .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + 8, + } }, + else => return self.fail("TODO implement slice_len for {f}", .{src_mcv}), + }; + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + switch (src_mcv) { + .register_pair => |regs| try self.freeValue(.{ .register = regs[0] }), + .load_frame => {}, + else => unreachable, + } + break :result len_mcv; + } + + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{}); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const src_ty = self.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const dst_ty = self.typeOfIndex(inst); + const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_reg + else + try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu)); + try self.asmRegisterMemory( + .{ ._, .lea }, + registerAlias(dst_reg, dst_abi_size), + .{ + .base = .{ .reg = src_reg }, + .mod = .{ .rm = .{ .disp = 8 } }, + }, + ); + + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dst_ty = self.typeOfIndex(inst); + const opt_mcv = try self.resolveInst(ty_op.operand); + + const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) + opt_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv); + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register { + const reg: Register = blk: { + switch (index) { + .immediate => |imm| { + // Optimisation: if index MCValue is an immediate, we can multiply in `comptime` + // and set the register directly to the scaled offset as an immediate. + const reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size }, .{}); + break :blk reg; + }, + else => { + const reg = try self.copyToTmpRegister(index_ty, index); + try self.genIntMulComplexOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size }); + break :blk reg; + }, + } + }; + return reg; +} + +fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const slice_ty = self.typeOf(lhs); + const slice_mcv = try self.resolveInst(lhs); + const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (slice_mcv_lock) |lock| self.register_manager.unlockReg(lock); + + const elem_ty = slice_ty.childType(zcu); + const elem_size = elem_ty.abiSize(zcu); + const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); + + const index_ty = self.typeOf(rhs); + const index_mcv = try self.resolveInst(rhs); + const index_mcv_lock: ?RegisterLock = switch (index_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_mcv_lock) |lock| self.register_manager.unlockReg(lock); + + const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size); + const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); + defer self.register_manager.unlockReg(offset_reg_lock); + + const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + try self.genSetReg(addr_reg, .usize, slice_mcv, .{}); + // TODO we could allocate register here, but need to expect addr register and potentially + // offset register. + try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{ + .register = offset_reg, + }); + return MCValue{ .register = addr_reg.to64() }; +} + +fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const result: MCValue = result: { + const elem_ty = self.typeOfIndex(inst); + if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const slice_ty = self.typeOf(bin_op.lhs); + const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu); + const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs); + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.load(dst_mcv, slice_ptr_field_type, elem_ptr); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs); + return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); +} + +fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const result: MCValue = result: { + const array_ty = self.typeOf(bin_op.lhs); + const elem_ty = array_ty.childType(zcu); + + const array_mcv = try self.resolveInst(bin_op.lhs); + const array_lock: ?RegisterLock = switch (array_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (array_lock) |lock| self.register_manager.unlockReg(lock); + + const index_ty = self.typeOf(bin_op.rhs); + const index_mcv = try self.resolveInst(bin_op.rhs); + const index_lock = switch (index_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_lock) |lock| self.register_manager.unlockReg(lock); + + try self.spillEflagsIfOccupied(); + if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) { + const array_mat_mcv: MCValue = switch (array_mcv) { + else => array_mcv, + .register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) }, + }; + const array_mat_lock = switch (array_mat_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (array_mat_lock) |lock| self.register_manager.unlockReg(lock); + + switch (array_mat_mcv) { + .register => |array_reg| switch (array_reg.class()) { + .general_purpose => switch (index_mcv) { + .immediate => |index_imm| try self.asmRegisterImmediate( + .{ ._, .bt }, + array_reg.to64(), + .u(index_imm), + ), + else => try self.asmRegisterRegister( + .{ ._, .bt }, + array_reg.to64(), + switch (index_mcv) { + .register => |index_reg| index_reg, + else => try self.copyToTmpRegister(index_ty, index_mcv), + }.to64(), + ), + }, + .sse => { + const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{}); + switch (index_mcv) { + .immediate => |index_imm| try self.asmMemoryImmediate( + .{ ._, .bt }, + .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = @intCast(index_imm / 64 * 8), + } }, + }, + .u(index_imm % 64), + ), + else => try self.asmMemoryRegister( + .{ ._, .bt }, + .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .qword } }, + }, + switch (index_mcv) { + .register => |index_reg| index_reg, + else => try self.copyToTmpRegister(index_ty, index_mcv), + }.to64(), + ), + } + }, + else => unreachable, + }, + .load_frame => switch (index_mcv) { + .immediate => |index_imm| try self.asmMemoryImmediate( + .{ ._, .bt }, + try array_mat_mcv.mem(self, .{ + .size = .qword, + .disp = @intCast(index_imm / 64 * 8), + }), + .u(index_imm % 64), + ), + else => try self.asmMemoryRegister( + .{ ._, .bt }, + try array_mat_mcv.mem(self, .{ .size = .qword }), + switch (index_mcv) { + .register => |index_reg| index_reg, + else => try self.copyToTmpRegister(index_ty, index_mcv), + }.to64(), + ), + }, + .memory, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => switch (index_mcv) { + .immediate => |index_imm| try self.asmMemoryImmediate( + .{ ._, .bt }, + .{ + .base = .{ + .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()), + }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = @intCast(index_imm / 64 * 8), + } }, + }, + .u(index_imm % 64), + ), + else => try self.asmMemoryRegister( + .{ ._, .bt }, + .{ + .base = .{ + .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()), + }, + .mod = .{ .rm = .{ .size = .qword } }, + }, + switch (index_mcv) { + .register => |index_reg| index_reg, + else => try self.copyToTmpRegister(index_ty, index_mcv), + }.to64(), + ), + }, + else => return self.fail("TODO airArrayElemVal for {s} of {f}", .{ + @tagName(array_mat_mcv), array_ty.fmt(pt), + }), + } + + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + try self.asmSetccRegister(.c, dst_reg.to8()); + break :result .{ .register = dst_reg }; + } + + const elem_abi_size = elem_ty.abiSize(zcu); + const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_lock); + + switch (array_mcv) { + .register => { + const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{}); + try self.asmRegisterMemory( + .{ ._, .lea }, + addr_reg, + .{ .base = .{ .frame = frame_index } }, + ); + }, + .load_frame => |frame_addr| try self.asmRegisterMemory( + .{ ._, .lea }, + addr_reg, + .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ .disp = frame_addr.off } }, + }, + ), + .memory, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => try self.genSetReg(addr_reg, .usize, array_mcv.address(), .{}), + else => return self.fail("TODO airArrayElemVal_val for {s} of {f}", .{ + @tagName(array_mcv), array_ty.fmt(pt), + }), + } + + const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); + const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); + defer self.register_manager.unlockReg(offset_lock); + + // TODO we could allocate register here, but need to expect addr register and potentially + // offset register. + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.genBinOpMir(.{ ._, .add }, .usize, .{ .register = addr_reg }, .{ .register = offset_reg }); + try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{}); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ptr_ty = self.typeOf(bin_op.lhs); + + // this is identical to the `airPtrElemPtr` codegen expect here an + // additional `mov` is needed at the end to get the actual value + + const result = result: { + const elem_ty = ptr_ty.elemType2(zcu); + if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; + + const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); + const index_ty = self.typeOf(bin_op.rhs); + const index_mcv = try self.resolveInst(bin_op.rhs); + const index_lock = switch (index_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_lock) |lock| self.register_manager.unlockReg(lock); + + const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); + const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); + defer self.register_manager.unlockReg(offset_lock); + + const ptr_mcv = try self.resolveInst(bin_op.lhs); + const elem_ptr_reg = if (ptr_mcv.isRegister() and self.liveness.operandDies(inst, 0)) + ptr_mcv.register + else + try self.copyToTmpRegister(ptr_ty, ptr_mcv); + const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg); + defer self.register_manager.unlockReg(elem_ptr_lock); + try self.asmRegisterRegister( + .{ ._, .add }, + elem_ptr_reg, + offset_reg, + ); + + const dst_mcv = try self.allocRegOrMem(inst, true); + const dst_lock = switch (dst_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg }); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + + const result = result: { + const elem_ptr_ty = self.typeOfIndex(inst); + const base_ptr_ty = self.typeOf(extra.lhs); + + const base_ptr_mcv = try self.resolveInst(extra.lhs); + const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (base_ptr_lock) |lock| self.register_manager.unlockReg(lock); + + if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { + break :result if (self.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) + base_ptr_mcv + else + try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); + } + + const elem_ty = base_ptr_ty.elemType2(zcu); + const elem_abi_size = elem_ty.abiSize(zcu); + const index_ty = self.typeOf(extra.rhs); + const index_mcv = try self.resolveInst(extra.rhs); + const index_lock: ?RegisterLock = switch (index_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (index_lock) |lock| self.register_manager.unlockReg(lock); + + const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); + const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); + defer self.register_manager.unlockReg(offset_reg_lock); + + const dst_mcv = try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); + try self.genBinOpMir(.{ ._, .add }, elem_ptr_ty, dst_mcv, .{ .register = offset_reg }); + + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); +} + +fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ptr_union_ty = self.typeOf(bin_op.lhs); + const union_ty = ptr_union_ty.childType(zcu); + const tag_ty = self.typeOf(bin_op.rhs); + const layout = union_ty.unionGetLayout(zcu); + + if (layout.tag_size == 0) { + return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); + } + + const ptr = try self.resolveInst(bin_op.lhs); + const ptr_lock: ?RegisterLock = switch (ptr) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const tag = try self.resolveInst(bin_op.rhs); + const tag_lock: ?RegisterLock = switch (tag) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (tag_lock) |lock| self.register_manager.unlockReg(lock); + + const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align.compare(.lt, layout.payload_align)) blk: { + // TODO reusing the operand + const reg = try self.copyToTmpRegister(ptr_union_ty, ptr); + try self.genBinOpMir( + .{ ._, .add }, + ptr_union_ty, + .{ .register = reg }, + .{ .immediate = layout.payload_size }, + ); + break :blk MCValue{ .register = reg }; + } else ptr; + + const ptr_tag_ty = try pt.adjustPtrTypeChild(ptr_union_ty, tag_ty); + try self.store(ptr_tag_ty, adjusted_ptr, tag, .{}); + + return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const tag_ty = self.typeOfIndex(inst); + const union_ty = self.typeOf(ty_op.operand); + const layout = union_ty.unionGetLayout(zcu); + + if (layout.tag_size == 0) { + return self.finishAir(inst, .none, .{ ty_op.operand, .none, .none }); + } + + // TODO reusing the operand + const operand = try self.resolveInst(ty_op.operand); + const operand_lock: ?RegisterLock = switch (operand) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); + + const tag_abi_size = tag_ty.abiSize(zcu); + const dst_mcv: MCValue = blk: { + switch (operand) { + .load_frame => |frame_addr| { + if (tag_abi_size <= 8) { + const off: i32 = @intCast(layout.tagOffset()); + break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{ + .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off }, + }); + } + + return self.fail( + "TODO implement get_union_tag for ABI larger than 8 bytes and operand {f}", + .{operand}, + ); + }, + .register => { + const shift: u6 = @intCast(layout.tagOffset() * 8); + const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand); + try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, result, .u8, .{ .immediate = shift }); + break :blk MCValue{ + .register = registerAlias(result.register, @intCast(layout.tag_size)), + }; + }, + else => return self.fail("TODO implement get_union_tag for {f}", .{operand}), + } + }; + + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = result: { + try self.spillEflagsIfOccupied(); + + const dst_ty = self.typeOfIndex(inst); + const src_ty = self.typeOf(ty_op.operand); + if (src_ty.zigTypeTag(zcu) == .vector) return self.fail("TODO implement airClz for {f}", .{ + src_ty.fmt(pt), + }); + + const src_mcv = try self.resolveInst(ty_op.operand); + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, + else => src_mcv, + }; + const mat_src_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const abi_size: u31 = @intCast(src_ty.abiSize(zcu)); + const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); + const has_lzcnt = self.hasFeature(.lzcnt); + if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) { + const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { + .load_frame => |src_frame_addr| src_frame_addr, + else => { + const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); + try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); + break :src_frame_addr .{ .index = src_frame_addr }; + }, + }; + + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; + const extra_bits = abi_size * 8 - src_bits; + + const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); + defer self.register_manager.unlockReg(index_lock); + + try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len)); + switch (extra_bits) { + 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()), + else => try self.asmRegisterImmediate( + .{ ._, .mov }, + dst_reg.to32(), + .s(@as(i32, extra_bits) - 1), + ), + } + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32()); + const zero = try self.asmJccReloc(.z, undefined); + if (self.hasFeature(.slow_incdec)) { + try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .de }, index_reg.to32()); + } + try self.asmMemoryImmediate(.{ ._, .cmp }, .{ + .base = .{ .frame = src_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = index_reg.to64(), + .scale = .@"8", + .disp = src_frame_addr.off, + } }, + }, .u(0)); + _ = try self.asmJccReloc(.e, loop); + try self.asmRegisterMemory(.{ ._r, .bs }, dst_reg.to64(), .{ + .base = .{ .frame = src_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = index_reg.to64(), + .scale = .@"8", + .disp = src_frame_addr.off, + } }, + }); + self.performReloc(zero); + try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6)); + try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32()); + try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1)); + try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32()); + break :result dst_mcv; + } + + if (has_lzcnt) { + if (src_bits <= 8) { + const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); + try self.truncateRegister(src_ty, wide_reg); + try self.genBinOpMir(.{ ._, .lzcnt }, .u32, dst_mcv, .{ .register = wide_reg }); + try self.genBinOpMir( + .{ ._, .sub }, + dst_ty, + dst_mcv, + .{ .immediate = 32 - src_bits }, + ); + } else if (src_bits <= 64) { + try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv); + const extra_bits = self.regExtraBits(src_ty); + if (extra_bits > 0) { + try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits }); + } + } else { + assert(src_bits <= 128); + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_mcv = MCValue{ .register = tmp_reg }; + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.genBinOpMir(.{ ._, .lzcnt }, .u64, dst_mcv, if (mat_src_mcv.isBase()) + mat_src_mcv + else + .{ .register = mat_src_mcv.register_pair[0] }); + try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 }); + try self.genBinOpMir(.{ ._, .lzcnt }, .u64, tmp_mcv, if (mat_src_mcv.isBase()) + mat_src_mcv.address().offset(8).deref() + else + .{ .register = mat_src_mcv.register_pair[1] }); + try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32()); + + if (src_bits < 128) try self.genBinOpMir( + .{ ._, .sub }, + dst_ty, + dst_mcv, + .{ .immediate = 128 - src_bits }, + ); + } + break :result dst_mcv; + } + + assert(src_bits <= 64); + const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2); + if (std.math.isPowerOfTwo(src_bits)) { + const imm_reg = try self.copyToTmpRegister(dst_ty, .{ + .immediate = src_bits ^ (src_bits - 1), + }); + const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg); + defer self.register_manager.unlockReg(imm_lock); + + if (src_bits <= 8) { + const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); + const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); + defer self.register_manager.unlockReg(wide_lock); + + try self.truncateRegister(src_ty, wide_reg); + try self.genBinOpMir(.{ ._r, .bs }, .u16, dst_mcv, .{ .register = wide_reg }); + } else try self.genBinOpMir(.{ ._r, .bs }, src_ty, dst_mcv, mat_src_mcv); + + try self.asmCmovccRegisterRegister( + .z, + registerAlias(dst_reg, cmov_abi_size), + registerAlias(imm_reg, cmov_abi_size), + ); + + try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 }); + } else { + const imm_reg = try self.copyToTmpRegister(dst_ty, .{ + .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)), + }); + const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg); + defer self.register_manager.unlockReg(imm_lock); + + const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); + const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); + defer self.register_manager.unlockReg(wide_lock); + + try self.truncateRegister(src_ty, wide_reg); + try self.genBinOpMir( + .{ ._r, .bs }, + if (src_bits <= 8) .u16 else src_ty, + dst_mcv, + .{ .register = wide_reg }, + ); + + try self.asmCmovccRegisterRegister( + .nz, + registerAlias(imm_reg, cmov_abi_size), + registerAlias(dst_reg, cmov_abi_size), + ); + + try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 }, .{}); + try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg }); + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result = result: { + try self.spillEflagsIfOccupied(); + + const dst_ty = self.typeOfIndex(inst); + const src_ty = self.typeOf(ty_op.operand); + if (src_ty.zigTypeTag(zcu) == .vector) return self.fail("TODO implement airCtz for {f}", .{ + src_ty.fmt(pt), + }); + + const src_mcv = try self.resolveInst(ty_op.operand); + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, + else => src_mcv, + }; + const mat_src_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const abi_size: u31 = @intCast(src_ty.abiSize(zcu)); + const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); + const has_bmi = self.hasFeature(.bmi); + if (src_bits > @as(u32, if (has_bmi) 128 else 64)) { + const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { + .load_frame => |src_frame_addr| src_frame_addr, + else => { + const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); + try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); + break :src_frame_addr .{ .index = src_frame_addr }; + }, + }; + + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; + const extra_bits = abi_size * 8 - src_bits; + + const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); + defer self.register_manager.unlockReg(index_lock); + + try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1)); + switch (extra_bits) { + 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()), + 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()), + else => try self.asmRegisterImmediate( + .{ ._, .mov }, + dst_reg.to32(), + .s(-@as(i32, extra_bits)), + ), + } + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + if (self.hasFeature(.slow_incdec)) { + try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, index_reg.to32()); + } + try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len)); + const zero = try self.asmJccReloc(.nb, undefined); + try self.asmMemoryImmediate(.{ ._, .cmp }, .{ + .base = .{ .frame = src_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = index_reg.to64(), + .scale = .@"8", + .disp = src_frame_addr.off, + } }, + }, .u(0)); + _ = try self.asmJccReloc(.e, loop); + try self.asmRegisterMemory(.{ ._f, .bs }, dst_reg.to64(), .{ + .base = .{ .frame = src_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = index_reg.to64(), + .scale = .@"8", + .disp = src_frame_addr.off, + } }, + }); + self.performReloc(zero); + try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6)); + try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32()); + break :result dst_mcv; + } + + const wide_ty: Type = if (src_bits <= 8) .u16 else src_ty; + if (has_bmi) { + if (src_bits <= 64) { + const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0); + const masked_mcv = if (extra_bits > 0) masked: { + const tmp_mcv = tmp: { + if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) + break :tmp src_mcv; + try self.genSetReg(dst_reg, wide_ty, src_mcv, .{}); + break :tmp dst_mcv; + }; + try self.genBinOpMir( + .{ ._, .@"or" }, + wide_ty, + tmp_mcv, + .{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) << + @intCast(src_bits) }, + ); + break :masked tmp_mcv; + } else mat_src_mcv; + try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv); + } else { + assert(src_bits <= 128); + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_mcv = MCValue{ .register = tmp_reg }; + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) + mat_src_mcv + else + .{ .register = mat_src_mcv.register_pair[0] }; + const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) + mat_src_mcv.address().offset(8).deref() + else + .{ .register = mat_src_mcv.register_pair[1] }; + const masked_mcv = if (src_bits < 128) masked: { + try self.genCopy(.u64, dst_mcv, hi_mat_src_mcv, .{}); + try self.genBinOpMir( + .{ ._, .@"or" }, + .u64, + dst_mcv, + .{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) }, + ); + break :masked dst_mcv; + } else hi_mat_src_mcv; + try self.genBinOpMir(.{ ._, .tzcnt }, .u64, dst_mcv, masked_mcv); + try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 }); + try self.genBinOpMir(.{ ._, .tzcnt }, .u64, tmp_mcv, lo_mat_src_mcv); + try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32()); + } + break :result dst_mcv; + } + + assert(src_bits <= 64); + const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits }); + const width_lock = self.register_manager.lockRegAssumeUnused(width_reg); + defer self.register_manager.unlockReg(width_lock); + + if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) { + const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); + const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); + defer self.register_manager.unlockReg(wide_lock); + + try self.truncateRegister(src_ty, wide_reg); + try self.genBinOpMir(.{ ._f, .bs }, wide_ty, dst_mcv, .{ .register = wide_reg }); + } else try self.genBinOpMir(.{ ._f, .bs }, src_ty, dst_mcv, mat_src_mcv); + + const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2); + try self.asmCmovccRegisterRegister( + .z, + registerAlias(dst_reg, cmov_abi_size), + registerAlias(width_reg, cmov_abi_size), + ); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const result: MCValue = result: { + try self.spillEflagsIfOccupied(); + + const src_ty = self.typeOf(ty_op.operand); + const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); + if (src_ty.zigTypeTag(zcu) == .vector or src_abi_size > 16) + return self.fail("TODO implement airPopCount for {f}", .{src_ty.fmt(pt)}); + const src_mcv = try self.resolveInst(ty_op.operand); + + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, + else => src_mcv, + }; + const mat_src_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); + + if (src_abi_size <= 8) { + const dst_contains_src = + src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv); + const dst_reg = if (dst_contains_src) + src_mcv.getReg().? + else + try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genPopCount(dst_reg, src_ty, mat_src_mcv, dst_contains_src); + break :result .{ .register = dst_reg }; + } + + assert(src_abi_size > 8 and src_abi_size <= 16); + const tmp_regs = try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp); + const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); + defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); + + try self.genPopCount(tmp_regs[0], .usize, if (mat_src_mcv.isBase()) + mat_src_mcv + else + .{ .register = mat_src_mcv.register_pair[0] }, false); + const src_info = src_ty.intInfo(zcu); + const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1); + try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase()) + mat_src_mcv.address().offset(8).deref() + else + .{ .register = mat_src_mcv.register_pair[1] }, false); + try self.asmRegisterRegister(.{ ._, .add }, tmp_regs[0].to8(), tmp_regs[1].to8()); + break :result .{ .register = tmp_regs[0] }; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn genPopCount( + self: *CodeGen, + dst_reg: Register, + src_ty: Type, + src_mcv: MCValue, + dst_contains_src: bool, +) !void { + const pt = self.pt; + + const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu)); + if (self.hasFeature(.popcnt)) return self.genBinOpMir( + .{ ._, .popcnt }, + if (src_abi_size > 1) src_ty else .u32, + .{ .register = dst_reg }, + if (src_abi_size > 1) src_mcv else src: { + if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{}); + try self.truncateRegister(try src_ty.toUnsigned(pt), dst_reg); + break :src .{ .register = dst_reg }; + }, + ); + + const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8); + const imm_0_1: Immediate = .u(mask / 0b1_1); + const imm_00_11: Immediate = .u(mask / 0b01_01); + const imm_0000_1111: Immediate = .u(mask / 0b0001_0001); + const imm_0000_0001: Immediate = .u(mask / 0b1111_1111); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const dst = registerAlias(dst_reg, src_abi_size); + const tmp = registerAlias(tmp_reg, src_abi_size); + const imm = if (src_abi_size > 4) + try self.register_manager.allocReg(null, abi.RegisterClass.gp) + else + undefined; + + if (!dst_contains_src) try self.genSetReg(dst, src_ty, src_mcv, .{}); + // dst = operand + try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); + // tmp = operand + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1)); + // tmp = operand >> 1 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1); + try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); + } else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1); + // tmp = (operand >> 1) & 0x55...55 + try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp); + // dst = temp1 = operand - ((operand >> 1) & 0x55...55) + try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); + // tmp = temp1 + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2)); + // dst = temp1 >> 2 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11); + try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); + try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); + } else { + try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11); + try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11); + } + // tmp = temp1 & 0x33...33 + // dst = (temp1 >> 2) & 0x33...33 + try self.asmRegisterRegister(.{ ._, .add }, tmp, dst); + // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33) + try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp); + // dst = temp2 + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4)); + // tmp = temp2 >> 4 + try self.asmRegisterRegister(.{ ._, .add }, dst, tmp); + // dst = temp2 + (temp2 >> 4) + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111); + try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001); + try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); + try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp); + } else { + try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111); + if (src_abi_size > 1) { + try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001); + } + } + // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f + // dst = temp3 * 0x01...01 + if (src_abi_size > 1) { + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8)); + } + // dst = (temp3 * 0x01...01) >> (bits - 8) +} + +fn genByteSwap( + self: *CodeGen, + inst: Air.Inst.Index, + src_ty: Type, + src_mcv: MCValue, + mem_ok: bool, +) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const has_movbe = self.hasFeature(.movbe); + + if (src_ty.zigTypeTag(zcu) == .vector) return self.fail( + "TODO implement genByteSwap for {f}", + .{src_ty.fmt(pt)}, + ); + + const src_lock = switch (src_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const abi_size: u32 = @intCast(src_ty.abiSize(zcu)); + switch (abi_size) { + 0 => unreachable, + 1 => return if ((mem_ok or src_mcv.isRegister()) and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv), + 2 => if ((mem_ok or src_mcv.isRegister()) and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + { + try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 }); + return src_mcv; + }, + 3...8 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + try self.genUnOpMir(.{ .b_, .swap }, src_ty, src_mcv); + return src_mcv; + }, + 9...16 => { + const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) { + .register => { + const frame_index = try self.allocFrameIndex(.initSpill(src_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{}); + break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } }; + }, + .register_pair => |src_regs| if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + for (src_regs) |src_reg| try self.asmRegister(.{ .b_, .swap }, src_reg.to64()); + return .{ .register_pair = .{ src_regs[1], src_regs[0] } }; + } else src_mcv, + else => src_mcv, + }; + + const dst_regs = + try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp); + const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); + defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); + + for (dst_regs, 0..) |dst_reg, limb_index| { + if (mat_src_mcv.isBase()) { + try self.asmRegisterMemory( + .{ if (has_movbe) ._be else ._, .mov }, + dst_reg.to64(), + try mat_src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }), + ); + if (!has_movbe) try self.asmRegister(.{ .b_, .swap }, dst_reg.to64()); + } else { + try self.asmRegisterRegister( + .{ ._, .mov }, + dst_reg.to64(), + mat_src_mcv.register_pair[limb_index].to64(), + ); + try self.asmRegister(.{ .b_, .swap }, dst_reg.to64()); + } + } + return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } }; + }, + else => { + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; + + const temp_regs = + try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); + const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs); + defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32()); + try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1)); + + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterMemory( + .{ if (has_movbe) ._be else ._, .mov }, + temp_regs[2].to64(), + .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[0].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off, + } }, + }, + ); + try self.asmRegisterMemory( + .{ if (has_movbe) ._be else ._, .mov }, + temp_regs[3].to64(), + .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[1].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off, + } }, + }, + ); + if (!has_movbe) { + try self.asmRegister(.{ .b_, .swap }, temp_regs[2].to64()); + try self.asmRegister(.{ .b_, .swap }, temp_regs[3].to64()); + } + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[0].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off, + } }, + }, temp_regs[3].to64()); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[1].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off, + } }, + }, temp_regs[2].to64()); + if (self.hasFeature(.slow_incdec)) { + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32()); + try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32()); + } + try self.asmRegisterRegister(.{ ._, .cmp }, temp_regs[0].to32(), temp_regs[1].to32()); + _ = try self.asmJccReloc(.be, loop); + return dst_mcv; + }, + } + + const dst_mcv: MCValue = if (mem_ok and has_movbe and src_mcv.isRegister()) + try self.allocRegOrMem(inst, true) + else + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) }; + if (dst_mcv.getReg()) |dst_reg| { + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register); + defer self.register_manager.unlockReg(dst_lock); + + try self.genSetReg(dst_reg, src_ty, src_mcv, .{}); + switch (abi_size) { + else => unreachable, + 2 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }), + 3...8 => try self.genUnOpMir(.{ .b_, .swap }, src_ty, dst_mcv), + } + } else try self.genBinOpMir(.{ ._be, .mov }, src_ty, dst_mcv, src_mcv); + return dst_mcv; +} + +fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const src_ty = self.typeOf(ty_op.operand); + const src_bits: u32 = @intCast(src_ty.bitSize(zcu)); + const src_mcv = try self.resolveInst(ty_op.operand); + + const dst_mcv = try self.genByteSwap(inst, src_ty, src_mcv, true); + try self.genShiftBinOpMir( + .{ ._r, switch (if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned) { + .signed => .sa, + .unsigned => .sh, + } }, + src_ty, + dst_mcv, + if (src_bits > 256) .u16 else .u8, + .{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits }, + ); + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const src_ty = self.typeOf(ty_op.operand); + const abi_size: u32 = @intCast(src_ty.abiSize(zcu)); + const bit_size: u32 = @intCast(src_ty.bitSize(zcu)); + const src_mcv = try self.resolveInst(ty_op.operand); + + const dst_mcv = try self.genByteSwap(inst, src_ty, src_mcv, false); + const dst_locks: [2]?RegisterLock = switch (dst_mcv) { + .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null }, + .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs), + else => unreachable, + }; + defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const limb_abi_size: u32 = @min(abi_size, 8); + const tmp = registerAlias(tmp_reg, limb_abi_size); + const imm = if (limb_abi_size > 4) + try self.register_manager.allocReg(null, abi.RegisterClass.gp) + else + undefined; + + const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8); + const imm_0000_1111: Immediate = .u(mask / 0b0001_0001); + const imm_00_11: Immediate = .u(mask / 0b01_01); + const imm_0_1: Immediate = .u(mask / 0b1_1); + + for (dst_mcv.getRegs()) |dst_reg| { + const dst = registerAlias(dst_reg, limb_abi_size); + + // dst = temp1 = bswap(operand) + try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); + // tmp = temp1 + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4)); + // dst = temp1 >> 4 + if (limb_abi_size > 4) { + try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111); + try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); + try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); + } else { + try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111); + try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111); + } + // tmp = temp1 & 0x0f...0f + // dst = (temp1 >> 4) & 0x0f...0f + try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4)); + // tmp = (temp1 & 0x0f...0f) << 4 + try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp); + // dst = temp2 = ((temp1 >> 4) & 0x0f...0f) | ((temp1 & 0x0f...0f) << 4) + try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); + // tmp = temp2 + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2)); + // dst = temp2 >> 2 + if (limb_abi_size > 4) { + try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11); + try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); + try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); + } else { + try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11); + try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11); + } + // tmp = temp2 & 0x33...33 + // dst = (temp2 >> 2) & 0x33...33 + try self.asmRegisterMemory( + .{ ._, .lea }, + if (limb_abi_size > 4) tmp.to64() else tmp.to32(), + .{ + .base = .{ .reg = dst.to64() }, + .mod = .{ .rm = .{ + .index = tmp.to64(), + .scale = .@"4", + } }, + }, + ); + // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2) + try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp); + // dst = temp3 + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1)); + // tmp = temp3 >> 1 + if (limb_abi_size > 4) { + try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1); + try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm); + try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm); + } else { + try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1); + try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1); + } + // dst = temp3 & 0x55...55 + // tmp = (temp3 >> 1) & 0x55...55 + try self.asmRegisterMemory( + .{ ._, .lea }, + if (limb_abi_size > 4) dst.to64() else dst.to32(), + .{ + .base = .{ .reg = tmp.to64() }, + .mod = .{ .rm = .{ + .index = dst.to64(), + .scale = .@"2", + } }, + }, + ); + // dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1) + } + + const extra_bits = abi_size * 8 - bit_size; + const signedness: std.builtin.Signedness = + if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned; + if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) { + .signed => .{ ._r, .sa }, + .unsigned => .{ ._r, .sh }, + }, src_ty, dst_mcv, .u8, .{ .immediate = extra_bits }); + + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn floatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag, operand: Air.Inst.Ref, ty: Type) !void { + const pt = self.pt; + const zcu = pt.zcu; + + const result = result: { + const scalar_bits = ty.scalarType(zcu).floatBits(self.target); + if (scalar_bits == 80) { + if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement floatSign for {f}", .{ + ty.fmt(pt), + }); + + const src_mcv = try self.resolveInst(operand); + const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv: MCValue = .{ .register = .st0 }; + if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv)) + try self.register_manager.getKnownReg(.st0, inst); + + try self.genCopy(ty, dst_mcv, src_mcv, .{}); + switch (tag) { + .neg => try self.asmOpOnly(.{ .f_, .chs }), + .abs => try self.asmOpOnly(.{ .f_, .abs }), + else => unreachable, + } + break :result dst_mcv; + } + + const abi_size: u32 = switch (ty.abiSize(zcu)) { + 1...16 => 16, + 17...32 => 32, + else => return self.fail("TODO implement floatSign for {f}", .{ + ty.fmt(pt), + }), + }; + + const src_mcv = try self.resolveInst(operand); + const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv: MCValue = if (src_mcv.isRegister() and + self.reuseOperand(inst, operand, 0, src_mcv)) + src_mcv + else if (self.hasFeature(.avx)) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const vec_ty = try pt.vectorType(.{ + .len = @divExact(abi_size * 8, scalar_bits), + .child = (try pt.intType(.signed, scalar_bits)).ip_index, + }); + + const sign_mcv = try self.lowerValue(switch (tag) { + .neg => try vec_ty.minInt(pt, vec_ty), + .abs => try vec_ty.maxInt(pt, vec_ty), + else => unreachable, + }); + const sign_mem: Memory = if (sign_mcv.isBase()) + try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) }) + else + .{ + .base = .{ .reg = try self.copyToTmpRegister(.usize, sign_mcv.address()) }, + .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, + }; + + if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory( + switch (scalar_bits) { + 16, 128 => if (abi_size <= 16 or self.hasFeature(.avx2)) switch (tag) { + .neg => .{ .vp_, .xor }, + .abs => .{ .vp_, .@"and" }, + else => unreachable, + } else switch (tag) { + .neg => .{ .v_ps, .xor }, + .abs => .{ .v_ps, .@"and" }, + else => unreachable, + }, + 32 => switch (tag) { + .neg => .{ .v_ps, .xor }, + .abs => .{ .v_ps, .@"and" }, + else => unreachable, + }, + 64 => switch (tag) { + .neg => .{ .v_pd, .xor }, + .abs => .{ .v_pd, .@"and" }, + else => unreachable, + }, + 80 => return self.fail("TODO implement floatSign for {f}", .{ty.fmt(pt)}), + else => unreachable, + }, + registerAlias(dst_reg, abi_size), + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv), abi_size), + sign_mem, + ) else try self.asmRegisterMemory( + switch (scalar_bits) { + 16, 128 => switch (tag) { + .neg => .{ .p_, .xor }, + .abs => .{ .p_, .@"and" }, + else => unreachable, + }, + 32 => switch (tag) { + .neg => .{ ._ps, .xor }, + .abs => .{ ._ps, .@"and" }, + else => unreachable, + }, + 64 => switch (tag) { + .neg => .{ ._pd, .xor }, + .abs => .{ ._pd, .@"and" }, + else => unreachable, + }, + 80 => return self.fail("TODO implement floatSign for {f}", .{ty.fmt(pt)}), + else => unreachable, + }, + registerAlias(dst_reg, abi_size), + sign_mem, + ); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ operand, .none, .none }); +} + +fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ty = self.typeOf(un_op); + return self.floatSign(inst, tag, un_op, ty); +} + +fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: bits.RoundMode) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ty = self.typeOf(un_op); + + const result = result: { + switch (try self.genRoundLibcall(ty, .{ .air_ref = un_op }, mode)) { + .none => {}, + else => |dst_mcv| break :result dst_mcv, + } + + const src_mcv = try self.resolveInst(un_op); + const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + try self.genRound(ty, dst_reg, src_mcv, mode); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag { + const pt = self.pt; + const zcu = pt.zcu; + return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(self.target)) { + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (ty.childType(zcu).floatBits(self.target)) { + 32 => switch (ty.vectorLen(zcu)) { + 1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round }, + 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round }, + 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null, + else => null, + }, + 64 => switch (ty.vectorLen(zcu)) { + 1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round }, + 2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round }, + 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => null, + }, + else => unreachable, + } else null; +} + +fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: bits.RoundMode) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + if (self.getRoundTag(ty)) |_| return .none; + + if (ty.zigTypeTag(zcu) != .float) + return self.fail("TODO implement genRound for {f}", .{ty.fmt(pt)}); + + var sym_buf: ["__trunc?".len]u8 = undefined; + return try self.genCall(.{ .extern_func = .{ + .return_type = ty.toIntern(), + .param_types = &.{ty.toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "{s}{s}{s}", .{ + floatLibcAbiPrefix(ty), + switch (mode.direction) { + .down => "floor", + .up => "ceil", + .zero => "trunc", + else => unreachable, + }, + floatLibcAbiSuffix(ty), + }) catch unreachable, + } }, &.{ty}, &.{src_mcv}, .{}); +} + +fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: bits.RoundMode) !void { + const pt = self.pt; + const mir_tag = self.getRoundTag(ty) orelse { + const result = try self.genRoundLibcall(ty, src_mcv, mode); + return self.genSetReg(dst_reg, ty, result, .{}); + }; + const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); + const dst_alias = registerAlias(dst_reg, abi_size); + switch (mir_tag[0]) { + .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + mir_tag, + dst_alias, + dst_alias, + try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + mode.imm(), + ) else try self.asmRegisterRegisterRegisterImmediate( + mir_tag, + dst_alias, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv), abi_size), + mode.imm(), + ), + else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( + mir_tag, + dst_alias, + try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + mode.imm(), + ) else try self.asmRegisterRegisterImmediate( + mir_tag, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv), abi_size), + mode.imm(), + ), + } +} + +fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const ty = self.typeOf(ty_op.operand); + + const result: MCValue = result: { + const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(zcu)) { + else => null, + .int => switch (ty.abiSize(zcu)) { + 0 => unreachable, + 1...8 => { + try self.spillEflagsIfOccupied(); + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); + + try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv); + + const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2); + switch (src_mcv) { + .register => |val_reg| try self.asmCmovccRegisterRegister( + .l, + registerAlias(dst_mcv.register, cmov_abi_size), + registerAlias(val_reg, cmov_abi_size), + ), + .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( + .l, + registerAlias(dst_mcv.register, cmov_abi_size), + try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }), + ), + else => { + const val_reg = try self.copyToTmpRegister(ty, src_mcv); + try self.asmCmovccRegisterRegister( + .l, + registerAlias(dst_mcv.register, cmov_abi_size), + registerAlias(val_reg, cmov_abi_size), + ); + }, + } + break :result dst_mcv; + }, + 9...16 => { + try self.spillEflagsIfOccupied(); + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_mcv = if (src_mcv == .register_pair and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { + const dst_regs = try self.register_manager.allocRegs( + 2, + .{ inst, inst }, + abi.RegisterClass.gp, + ); + const dst_mcv: MCValue = .{ .register_pair = dst_regs }; + const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); + defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); + + try self.genCopy(ty, dst_mcv, src_mcv, .{}); + break :dst dst_mcv; + }; + const dst_regs = dst_mcv.register_pair; + const dst_locks = self.register_manager.lockRegs(2, dst_regs); + defer for (dst_locks) |dst_lock| if (dst_lock) |lock| + self.register_manager.unlockReg(lock); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); + try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63)); + try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg); + try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg); + try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg); + try self.asmRegisterRegister(.{ ._, .sbb }, dst_regs[1], tmp_reg); + + break :result dst_mcv; + }, + else => { + const abi_size: u31 = @intCast(ty.abiSize(zcu)); + const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable; + + const tmp_regs = + try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp); + const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs); + defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); + + try self.spillEflagsIfOccupied(); + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.allocRegOrMem(inst, false); + + try self.asmMemoryImmediate( + .{ ._, .cmp }, + try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }), + .u(0), + ); + const positive = try self.asmJccReloc(.ns, undefined); + + try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[0].to32(), tmp_regs[0].to32()); + try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[1].to8(), tmp_regs[1].to8()); + + const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32()); + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1)); + try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = tmp_regs[0].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off, + } }, + }); + try self.asmSetccRegister(.c, tmp_regs[1].to8()); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = tmp_regs[0].to64(), + .scale = .@"8", + .disp = dst_mcv.load_frame.off, + } }, + }, tmp_regs[2].to64()); + + if (self.hasFeature(.slow_incdec)) { + try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, tmp_regs[0].to32()); + } + try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len)); + _ = try self.asmJccReloc(.b, neg_loop); + + self.performReloc(positive); + break :result dst_mcv; + }, + }, + .float => return self.floatSign(inst, .abs, ty_op.operand, ty), + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + else => null, + .int => switch (ty.childType(zcu).intInfo(zcu).bits) { + else => null, + 8 => switch (ty.vectorLen(zcu)) { + else => null, + 1...16 => if (self.hasFeature(.avx)) + .{ .vp_b, .abs } + else if (self.hasFeature(.ssse3)) + .{ .p_b, .abs } + else + null, + 17...32 => if (self.hasFeature(.avx2)) .{ .vp_b, .abs } else null, + }, + 16 => switch (ty.vectorLen(zcu)) { + else => null, + 1...8 => if (self.hasFeature(.avx)) + .{ .vp_w, .abs } + else if (self.hasFeature(.ssse3)) + .{ .p_w, .abs } + else + null, + 9...16 => if (self.hasFeature(.avx2)) .{ .vp_w, .abs } else null, + }, + 32 => switch (ty.vectorLen(zcu)) { + else => null, + 1...4 => if (self.hasFeature(.avx)) + .{ .vp_d, .abs } + else if (self.hasFeature(.ssse3)) + .{ .p_d, .abs } + else + null, + 5...8 => if (self.hasFeature(.avx2)) .{ .vp_d, .abs } else null, + }, + }, + .float => return self.floatSign(inst, .abs, ty_op.operand, ty), + }, + }) orelse return self.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)}); + + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv.getReg().? + else + try self.register_manager.allocReg(inst, self.regSetForType(ty)); + const dst_alias = registerAlias(dst_reg, abi_size); + if (src_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + dst_alias, + try src_mcv.mem(self, .{ .size = self.memSize(ty) }), + ) else try self.asmRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv), abi_size), + ); + break :result .{ .register = dst_reg }; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ty = self.typeOf(un_op); + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + + const result: MCValue = result: { + switch (ty.zigTypeTag(zcu)) { + .float => { + const float_bits = ty.floatBits(self.target); + if (switch (float_bits) { + 16 => !self.hasFeature(.f16c), + 32, 64 => false, + 80, 128 => true, + else => unreachable, + }) { + var sym_buf: ["__sqrt?".len]u8 = undefined; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = ty.toIntern(), + .param_types = &.{ty.toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "{s}sqrt{s}", .{ + floatLibcAbiPrefix(ty), + floatLibcAbiSuffix(ty), + }) catch unreachable, + } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{}); + } + }, + else => {}, + } + + const src_mcv = try self.resolveInst(un_op); + const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, ty, src_mcv); + const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(self.target)) { + 16 => { + assert(self.hasFeature(.f16c)); + const mat_src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv); + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128()); + try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + break :result dst_mcv; + }, + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt }, + else => unreachable, + }, + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (ty.childType(zcu).floatBits(self.target)) { + 16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen(zcu)) { + 1 => { + try self.asmRegisterRegister( + .{ .v_ps, .cvtph2 }, + dst_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv)).to128(), + ); + try self.asmRegisterRegisterRegister( + .{ .v_ss, .sqrt }, + dst_reg, + dst_reg, + dst_reg, + ); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + break :result dst_mcv; + }, + 2...8 => { + const wide_reg = registerAlias(dst_reg, abi_size * 2); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_ps, .cvtph2 }, + wide_reg, + try src_mcv.mem(self, .{ .size = .fromSize( + @intCast(@divExact(wide_reg.bitSize(), 16)), + ) }), + ) else try self.asmRegisterRegister( + .{ .v_ps, .cvtph2 }, + wide_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv)).to128(), + ); + try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + wide_reg, + bits.RoundMode.imm(.{}), + ); + break :result dst_mcv; + }, + else => null, + } else null, + 32 => switch (ty.vectorLen(zcu)) { + 1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt }, + 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt }, + 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null, + else => null, + }, + 64 => switch (ty.vectorLen(zcu)) { + 1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt }, + 2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt }, + 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null, + else => null, + }, + 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }) orelse return self.fail("TODO implement airSqrt for {f}", .{ty.fmt(pt)}); + switch (mir_tag[0]) { + .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( + mir_tag, + dst_reg, + dst_reg, + try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_reg, + dst_reg, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv), abi_size), + ), + else => if (src_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + dst_reg, + try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + ) else try self.asmRegisterRegister( + mir_tag, + dst_reg, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv), abi_size), + ), + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ty = self.typeOf(un_op); + var sym_buf: ["__round?".len]u8 = undefined; + const result = try self.genCall(.{ .extern_func = .{ + .return_type = ty.toIntern(), + .param_types = &.{ty.toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "{s}{s}{s}", .{ + floatLibcAbiPrefix(ty), + switch (tag) { + .sin, + .cos, + .tan, + .exp, + .exp2, + .log, + .log2, + .log10, + .round, + => @tagName(tag), + else => unreachable, + }, + floatLibcAbiSuffix(ty), + }) catch unreachable, + } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{}); + return self.finishAir(inst, result, .{ un_op, .none, .none }); +} + +fn reuseOperand( + self: *CodeGen, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + op_index: Air.Liveness.OperandInt, + mcv: MCValue, +) bool { + return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); +} + +fn reuseOperandAdvanced( + self: *CodeGen, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + op_index: Air.Liveness.OperandInt, + mcv: MCValue, + maybe_tracked_inst: ?Air.Inst.Index, +) bool { + if (!self.liveness.operandDies(inst, op_index)) + return false; + + switch (mcv) { + .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| { + // If it's in the registers table, need to associate the register(s) with the + // new instruction. + if (maybe_tracked_inst) |tracked_inst| { + if (!self.register_manager.isRegFree(reg)) { + if (RegisterManager.indexOfRegIntoTracked(reg)) |index| { + self.register_manager.registers[index] = tracked_inst; + } + } + } else self.register_manager.freeReg(reg); + }, + .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, + else => return false, + } + switch (mcv) { + .eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst, + else => {}, + } + + // Prevent the operand deaths processing code from deallocating it. + self.reused_operands.set(op_index); + const op_inst = operand.toIndex().?; + self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst); + + return true; +} + +fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const dst_ty = ptr_ty.childType(zcu); + if (!dst_ty.hasRuntimeBitsIgnoreComptime(zcu)) return; + switch (ptr_mcv) { + .none, + .unreach, + .dead, + .undef, + .eflags, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + => unreachable, // not a valid pointer + .immediate, + .register, + .register_offset, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref(), .{}), + .memory, + .indirect, + .load_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => { + const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); + const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_lock); + + try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{}); + }, + .air_ref => |ptr_ref| try self.load(dst_mcv, ptr_ty, try self.resolveInst(ptr_ref)), + } +} + +fn store( + self: *CodeGen, + ptr_ty: Type, + ptr_mcv: MCValue, + src_mcv: MCValue, + opts: CopyOptions, +) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const src_ty = ptr_ty.childType(zcu); + if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return; + switch (ptr_mcv) { + .none, + .unreach, + .dead, + .undef, + .eflags, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + => unreachable, // not a valid pointer + .immediate, + .register, + .register_offset, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv, opts), + .memory, + .indirect, + .load_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => { + const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); + const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_lock); + + try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv, opts); + }, + .air_ref => |ptr_ref| try self.store(ptr_ty, try self.resolveInst(ptr_ref), src_mcv, opts), + } +} + +fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const src_ty = self.typeOf(src_air); + if (src_ty.zigTypeTag(zcu) == .vector) + return self.fail("TODO implement genUnOp for {f}", .{src_ty.fmt(pt)}); + + var src_mcv = try self.resolveInst(src_air); + switch (src_mcv) { + .eflags => |cc| switch (tag) { + .not => { + if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) + return .{ .eflags = cc.negate() }; + try self.spillEflagsIfOccupied(); + src_mcv = try self.resolveInst(src_air); + }, + else => {}, + }, + else => {}, + } + + const src_lock = switch (src_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv: MCValue = dst: { + if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) break :dst src_mcv; + + const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, maybe_inst, true); + try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); + break :dst dst_mcv; + }; + const dst_lock = switch (dst_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const abi_size: u16 = @intCast(src_ty.abiSize(zcu)); + switch (tag) { + .not => { + const limb_abi_size: u16 = @min(abi_size, 8); + const int_info: InternPool.Key.IntType = if (src_ty.ip_index == .bool_type) + .{ .signedness = .unsigned, .bits = 1 } + else + src_ty.intInfo(zcu); + var byte_off: i32 = 0; + while (byte_off * 8 < int_info.bits) : (byte_off += limb_abi_size) { + const limb_bits: u16 = @intCast(@min(switch (int_info.signedness) { + .signed => abi_size * 8, + .unsigned => int_info.bits, + } - byte_off * 8, limb_abi_size * 8)); + const limb_ty = try pt.intType(int_info.signedness, limb_bits); + const limb_mcv = switch (byte_off) { + 0 => dst_mcv, + else => dst_mcv.address().offset(byte_off).deref(), + }; + + if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) { + const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits); + try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask }); + } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv); + } + }, + .neg => { + try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv); + const bit_size = src_ty.intInfo(zcu).bits; + if (abi_size * 8 > bit_size) { + if (dst_mcv.isRegister()) { + try self.truncateRegister(src_ty, dst_mcv.getReg().?); + } else { + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref(); + try self.genSetReg(tmp_reg, .usize, hi_mcv, .{}); + try self.truncateRegister(src_ty, tmp_reg); + try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{}); + } + } + }, + else => unreachable, + } + return dst_mcv; +} + +fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void { + const pt = self.pt; + const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu)); + if (abi_size > 8) return self.fail("TODO implement {} for {f}", .{ mir_tag, dst_ty.fmt(pt) }); + switch (dst_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .register_offset, + .eflags, + .register_overflow, + .register_mask, + .indirect_load_frame, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // unmodifiable destination + .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), + .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented + .memory, .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => { + const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + + try self.genSetReg(addr_reg, .usize, dst_mcv.address(), .{}); + try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + } } }); + }, + .indirect, .load_frame => try self.asmMemory( + mir_tag, + try dst_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + ), + } +} + +/// Clobbers .rcx for non-immediate shift value. +fn genShiftBinOpMir( + self: *CodeGen, + tag: Mir.Inst.FixedTag, + lhs_ty: Type, + lhs_mcv: MCValue, + rhs_ty: Type, + rhs_mcv: MCValue, +) !void { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size: u31 = @intCast(lhs_ty.abiSize(zcu)); + const shift_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu)); + try self.spillEflagsIfOccupied(); + + if (abi_size > 16) { + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; + assert(shift_abi_size >= 1 and shift_abi_size <= 2); + + const rcx_lock: ?RegisterLock = switch (rhs_mcv) { + .immediate => |shift_imm| switch (shift_imm) { + 0 => return, + else => null, + }, + else => lock: { + if (switch (rhs_mcv) { + .register => |rhs_reg| rhs_reg.id() != Register.rcx.id(), + else => true, + }) { + self.register_manager.getRegAssumeFree(.rcx, null); + try self.genSetReg(.rcx, rhs_ty, rhs_mcv, .{}); + } + break :lock self.register_manager.lockReg(.rcx); + }, + }; + defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); + + const temp_regs = try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp); + const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs); + defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); + + switch (tag[0]) { + ._l => { + try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1)); + switch (rhs_mcv) { + .immediate => |shift_imm| try self.asmRegisterImmediate( + .{ ._, .mov }, + temp_regs[0].to32(), + .u(limbs_len - (shift_imm >> 6) - 1), + ), + else => { + try self.asmRegisterRegister( + .{ ._, .movzx }, + temp_regs[2].to32(), + registerAlias(.rcx, shift_abi_size), + ); + try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6))); + try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6)); + try self.asmRegisterRegister( + .{ ._, .mov }, + temp_regs[0].to32(), + temp_regs[1].to32(), + ); + try self.asmRegisterRegister( + .{ ._, .sub }, + temp_regs[0].to32(), + temp_regs[2].to32(), + ); + }, + } + }, + ._r => { + try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[1].to32(), temp_regs[1].to32()); + switch (rhs_mcv) { + .immediate => |shift_imm| try self.asmRegisterImmediate( + .{ ._, .mov }, + temp_regs[0].to32(), + .u(shift_imm >> 6), + ), + else => { + try self.asmRegisterRegister( + .{ ._, .movzx }, + temp_regs[0].to32(), + registerAlias(.rcx, shift_abi_size), + ); + try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6))); + try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6)); + }, + } + }, + else => unreachable, + } + + const slow_inc_dec = self.hasFeature(.slow_incdec); + if (switch (rhs_mcv) { + .immediate => |shift_imm| shift_imm >> 6 < limbs_len - 1, + else => true, + }) { + try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[2].to64(), .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[0].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off, + } }, + }); + const skip = switch (rhs_mcv) { + .immediate => undefined, + else => switch (tag[0]) { + ._l => try self.asmJccReloc(.z, undefined), + ._r => skip: { + try self.asmRegisterImmediate( + .{ ._, .cmp }, + temp_regs[0].to32(), + .u(limbs_len - 1), + ); + break :skip try self.asmJccReloc(.nb, undefined); + }, + else => unreachable, + }, + }; + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[3].to64(), .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[0].to64(), + .scale = .@"8", + .disp = switch (tag[0]) { + ._l => lhs_mcv.load_frame.off - 8, + ._r => lhs_mcv.load_frame.off + 8, + else => unreachable, + }, + } }, + }); + switch (rhs_mcv) { + .immediate => |shift_imm| try self.asmRegisterRegisterImmediate( + .{ switch (tag[0]) { + ._l => ._ld, + ._r => ._rd, + else => unreachable, + }, .sh }, + temp_regs[2].to64(), + temp_regs[3].to64(), + .u(shift_imm & std.math.maxInt(u6)), + ), + else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) { + ._l => ._ld, + ._r => ._rd, + else => unreachable, + }, .sh }, temp_regs[2].to64(), temp_regs[3].to64(), .cl), + } + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[1].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off, + } }, + }, temp_regs[2].to64()); + try self.asmRegisterRegister(.{ ._, .mov }, temp_regs[2].to64(), temp_regs[3].to64()); + switch (tag[0]) { + ._l => { + if (slow_inc_dec) { + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32()); + try self.asmRegister(.{ ._c, .de }, temp_regs[0].to32()); + } + _ = try self.asmJccReloc(.nz, loop); + }, + ._r => { + if (slow_inc_dec) { + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, temp_regs[1].to32()); + try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32()); + } + try self.asmRegisterImmediate( + .{ ._, .cmp }, + temp_regs[0].to32(), + .u(limbs_len - 1), + ); + _ = try self.asmJccReloc(.b, loop); + }, + else => unreachable, + } + switch (rhs_mcv) { + .immediate => {}, + else => self.performReloc(skip), + } + } else try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[2].to64(), .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = switch (tag[0]) { + ._l => lhs_mcv.load_frame.off, + ._r => lhs_mcv.load_frame.off + abi_size - 8, + else => unreachable, + }, + } }, + }); + switch (rhs_mcv) { + .immediate => |shift_imm| try self.asmRegisterImmediate( + tag, + temp_regs[2].to64(), + .u(shift_imm & std.math.maxInt(u6)), + ), + else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl), + } + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[1].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off, + } }, + }, temp_regs[2].to64()); + if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( + tag, + temp_regs[2].to64(), + .u(63), + ); + if (switch (rhs_mcv) { + .immediate => |shift_imm| shift_imm >> 6 > 0, + else => true, + }) { + const skip = switch (rhs_mcv) { + .immediate => undefined, + else => switch (tag[0]) { + ._l => skip: { + try self.asmRegisterRegister( + .{ ._, .@"test" }, + temp_regs[1].to32(), + temp_regs[1].to32(), + ); + break :skip try self.asmJccReloc(.z, undefined); + }, + ._r => skip: { + try self.asmRegisterImmediate( + .{ ._, .cmp }, + temp_regs[1].to32(), + .u(limbs_len - 1), + ); + break :skip try self.asmJccReloc(.nb, undefined); + }, + else => unreachable, + }, + }; + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + switch (tag[0]) { + ._l => if (slow_inc_dec) { + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32()); + }, + ._r => if (slow_inc_dec) { + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, temp_regs[1].to32()); + }, + else => unreachable, + } + if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[1].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off, + } }, + }, temp_regs[2].to64()) else try self.asmMemoryImmediate(.{ ._, .mov }, .{ + .base = .{ .frame = lhs_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .index = temp_regs[1].to64(), + .scale = .@"8", + .disp = lhs_mcv.load_frame.off, + } }, + }, .u(0)); + switch (tag[0]) { + ._l => _ = try self.asmJccReloc(.nz, loop), + ._r => { + try self.asmRegisterImmediate( + .{ ._, .cmp }, + temp_regs[1].to32(), + .u(limbs_len - 1), + ); + _ = try self.asmJccReloc(.b, loop); + }, + else => unreachable, + } + switch (rhs_mcv) { + .immediate => {}, + else => self.performReloc(skip), + } + } + return; + } + + assert(shift_abi_size == 1); + const shift_mcv: MCValue = shift: { + switch (rhs_mcv) { + .immediate => |shift_imm| switch (shift_imm) { + 0 => return, + else => break :shift rhs_mcv, + }, + .register => |rhs_reg| if (rhs_reg.id() == Register.rcx.id()) + break :shift rhs_mcv, + else => {}, + } + self.register_manager.getRegAssumeFree(.rcx, null); + try self.genSetReg(.cl, rhs_ty, rhs_mcv, .{}); + break :shift .{ .register = .rcx }; + }; + if (abi_size > 8) { + const info: struct { indices: [2]u31, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) { + ._l => .{ .indices = .{ 0, 1 }, .double_tag = .{ ._ld, .sh } }, + ._r => .{ .indices = .{ 1, 0 }, .double_tag = .{ ._rd, .sh } }, + else => unreachable, + }; + switch (lhs_mcv) { + .register_pair => |lhs_regs| switch (shift_mcv) { + .immediate => |shift_imm| if (shift_imm > 0 and shift_imm < 64) { + try self.asmRegisterRegisterImmediate( + info.double_tag, + lhs_regs[info.indices[1]], + lhs_regs[info.indices[0]], + .u(shift_imm), + ); + try self.asmRegisterImmediate( + tag, + lhs_regs[info.indices[0]], + .u(shift_imm), + ); + return; + } else { + assert(shift_imm < 128); + try self.asmRegisterRegister( + .{ ._, .mov }, + lhs_regs[info.indices[1]], + lhs_regs[info.indices[0]], + ); + if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( + tag, + lhs_regs[info.indices[0]], + .u(63), + ) else try self.asmRegisterRegister( + .{ ._, .xor }, + lhs_regs[info.indices[0]], + lhs_regs[info.indices[0]], + ); + if (shift_imm > 64) try self.asmRegisterImmediate( + tag, + lhs_regs[info.indices[1]], + .u(shift_imm - 64), + ); + return; + }, + .register => |shift_reg| { + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + if (tag[0] == ._r and tag[1] == .sa) { + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]); + try self.asmRegisterImmediate(tag, tmp_reg, .u(63)); + } else try self.asmRegisterRegister( + .{ ._, .xor }, + tmp_reg.to32(), + tmp_reg.to32(), + ); + try self.asmRegisterRegisterRegister( + info.double_tag, + lhs_regs[info.indices[1]], + lhs_regs[info.indices[0]], + shift_reg.to8(), + ); + try self.asmRegisterRegister( + tag, + lhs_regs[info.indices[0]], + shift_reg.to8(), + ); + try self.asmRegisterImmediate(.{ ._, .cmp }, shift_reg.to8(), .u(64)); + try self.asmCmovccRegisterRegister( + .ae, + lhs_regs[info.indices[1]], + lhs_regs[info.indices[0]], + ); + try self.asmCmovccRegisterRegister(.ae, lhs_regs[info.indices[0]], tmp_reg); + return; + }, + else => {}, + }, + .load_frame => |dst_frame_addr| { + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + switch (shift_mcv) { + .immediate => |shift_imm| if (shift_imm > 0 and shift_imm < 64) { + try self.asmRegisterMemory( + .{ ._, .mov }, + tmp_reg, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + ); + try self.asmMemoryRegisterImmediate( + info.double_tag, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[1] * 8, + } }, + }, + tmp_reg, + .u(shift_imm), + ); + try self.asmMemoryImmediate( + tag, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + .u(shift_imm), + ); + return; + } else { + assert(shift_imm < 128); + try self.asmRegisterMemory( + .{ ._, .mov }, + tmp_reg, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + ); + if (shift_imm > 64) try self.asmRegisterImmediate( + tag, + tmp_reg, + .u(shift_imm - 64), + ); + try self.asmMemoryRegister( + .{ ._, .mov }, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[1] * 8, + } }, + }, + tmp_reg, + ); + if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate( + tag, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + .u(63), + ) else { + try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32()); + try self.asmMemoryRegister( + .{ ._, .mov }, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + tmp_reg, + ); + } + return; + }, + .register => |shift_reg| { + const first_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const first_lock = self.register_manager.lockRegAssumeUnused(first_reg); + defer self.register_manager.unlockReg(first_lock); + + const second_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const second_lock = self.register_manager.lockRegAssumeUnused(second_reg); + defer self.register_manager.unlockReg(second_lock); + + try self.asmRegisterMemory( + .{ ._, .mov }, + first_reg, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + ); + try self.asmRegisterMemory( + .{ ._, .mov }, + second_reg, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[1] * 8, + } }, + }, + ); + if (tag[0] == ._r and tag[1] == .sa) { + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg); + try self.asmRegisterImmediate(tag, tmp_reg, .u(63)); + } else try self.asmRegisterRegister( + .{ ._, .xor }, + tmp_reg.to32(), + tmp_reg.to32(), + ); + try self.asmRegisterRegisterRegister( + info.double_tag, + second_reg, + first_reg, + shift_reg.to8(), + ); + try self.asmRegisterRegister(tag, first_reg, shift_reg.to8()); + try self.asmRegisterImmediate( + .{ ._, .cmp }, + shift_reg.to8(), + .u(64), + ); + try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg); + try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg); + try self.asmMemoryRegister( + .{ ._, .mov }, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[1] * 8, + } }, + }, + second_reg, + ); + try self.asmMemoryRegister( + .{ ._, .mov }, + .{ + .base = .{ .frame = dst_frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_frame_addr.off + info.indices[0] * 8, + } }, + }, + first_reg, + ); + return; + }, + else => {}, + } + }, + else => {}, + } + } else switch (lhs_mcv) { + .register => |lhs_reg| switch (shift_mcv) { + .immediate => |shift_imm| return self.asmRegisterImmediate( + tag, + registerAlias(lhs_reg, abi_size), + .u(shift_imm), + ), + .register => |shift_reg| return self.asmRegisterRegister( + tag, + registerAlias(lhs_reg, abi_size), + shift_reg.to8(), + ), + else => {}, + }, + .memory, .indirect, .load_frame => { + const lhs_mem: Memory = switch (lhs_mcv) { + .memory => |addr| .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse + return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ + @tagName(lhs_mcv), + @tagName(shift_mcv), + }), + } }, + }, + .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = reg_off.off, + } }, + }, + .load_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = frame_addr.off, + } }, + }, + else => unreachable, + }; + switch (shift_mcv) { + .immediate => |shift_imm| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)), + .register => |shift_reg| return self.asmMemoryRegister( + tag, + lhs_mem, + shift_reg.to8(), + ), + else => {}, + } + }, + else => {}, + } + return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ + @tagName(lhs_mcv), + @tagName(shift_mcv), + }); +} + +fn genBinOp( + self: *CodeGen, + maybe_inst: ?Air.Inst.Index, + air_tag: Air.Inst.Tag, + lhs_air: Air.Inst.Ref, + rhs_air: Air.Inst.Ref, +) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const lhs_ty = self.typeOf(lhs_air); + const rhs_ty = self.typeOf(rhs_air); + const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); + + if (lhs_ty.isRuntimeFloat()) libcall: { + const float_bits = lhs_ty.floatBits(self.target); + const type_needs_libcall = switch (float_bits) { + 16 => !self.hasFeature(.f16c), + 32, 64 => false, + 80, 128 => true, + else => unreachable, + }; + switch (air_tag) { + .rem, .mod => {}, + else => if (!type_needs_libcall) break :libcall, + } + var sym_buf: ["__mod?f3".len]u8 = undefined; + const sym = switch (air_tag) { + .add, + .sub, + .mul, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => std.fmt.bufPrint(&sym_buf, "__{s}{c}f3", .{ + @tagName(air_tag)[0..3], + floatCompilerRtAbiName(float_bits), + }), + .rem, .mod, .min, .max => std.fmt.bufPrint(&sym_buf, "{s}f{s}{s}", .{ + floatLibcAbiPrefix(lhs_ty), + switch (air_tag) { + .rem, .mod => "mod", + .min => "min", + .max => "max", + else => unreachable, + }, + floatLibcAbiSuffix(lhs_ty), + }), + else => return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + } catch unreachable; + const result = try self.genCall(.{ .extern_func = .{ + .return_type = lhs_ty.toIntern(), + .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() }, + .sym = sym, + } }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } }, .{}); + return switch (air_tag) { + .mod => result: { + const adjusted: MCValue = if (type_needs_libcall) adjusted: { + var add_sym_buf: ["__add?f3".len]u8 = undefined; + break :adjusted try self.genCall(.{ .extern_func = .{ + .return_type = lhs_ty.toIntern(), + .param_types = &.{ + lhs_ty.toIntern(), + rhs_ty.toIntern(), + }, + .sym = std.fmt.bufPrint(&add_sym_buf, "__add{c}f3", .{ + floatCompilerRtAbiName(float_bits), + }) catch unreachable, + } }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } }, .{}); + } else switch (float_bits) { + 16, 32, 64 => adjusted: { + const dst_reg = switch (result) { + .register => |reg| reg, + else => if (maybe_inst) |inst| + (try self.copyToRegisterWithInstTracking(inst, lhs_ty, result)).register + else + try self.copyToTmpRegister(lhs_ty, result), + }; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(rhs_air); + const src_mcv: MCValue = if (float_bits == 16) src: { + assert(self.hasFeature(.f16c)); + const tmp_reg = (try self.register_manager.allocReg( + null, + abi.RegisterClass.sse, + )).to128(); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_w, .insr }, + dst_reg, + dst_reg, + try rhs_mcv.mem(self, .{ .size = .word }), + .u(1), + ) else try self.asmRegisterRegisterRegister( + .{ .vp_, .unpcklwd }, + dst_reg, + dst_reg, + (if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, rhs_mcv)).to128(), + ); + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); + try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); + break :src .{ .register = tmp_reg }; + } else rhs_mcv; + + if (self.hasFeature(.avx)) { + const mir_tag: Mir.Inst.FixedTag = switch (float_bits) { + 16, 32 => .{ .v_ss, .add }, + 64 => .{ .v_sd, .add }, + else => unreachable, + }; + if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( + mir_tag, + dst_reg, + dst_reg, + try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_reg, + dst_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + } else { + const mir_tag: Mir.Inst.FixedTag = switch (float_bits) { + 32 => .{ ._ss, .add }, + 64 => .{ ._sd, .add }, + else => unreachable, + }; + if (src_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + dst_reg, + try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), + ) else try self.asmRegisterRegister( + mir_tag, + dst_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + } + + if (float_bits == 16) try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + break :adjusted .{ .register = dst_reg }; + }, + 80, 128 => return self.fail("TODO implement genBinOp for {s} of {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + else => unreachable, + }; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = lhs_ty.toIntern(), + .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() }, + .sym = sym, + } }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } }, .{}); + }, + .div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{ + .direction = switch (air_tag) { + .div_trunc => .zero, + .div_floor => .down, + else => unreachable, + }, + .precision = .inexact, + }), + else => result, + }; + } + + const sse_op = switch (lhs_ty.zigTypeTag(zcu)) { + else => false, + .float => true, + .vector => switch (lhs_ty.childType(zcu).toIntern()) { + .bool_type, .u1_type => false, + else => true, + }, + }; + if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and + lhs_ty.scalarType(zcu).floatBits(self.target) == 80) or + lhs_ty.abiSize(zcu) > self.vectorSize(.float))) + return self.fail("TODO implement genBinOp for {s} {f}", .{ @tagName(air_tag), lhs_ty.fmt(pt) }); + + const maybe_mask_reg = switch (air_tag) { + else => null, + .rem, .mod => unreachable, + .max, .min => if (lhs_ty.scalarType(zcu).isRuntimeFloat()) registerAlias( + if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: { + try self.register_manager.getKnownReg(.xmm0, null); + break :mask .xmm0; + } else try self.register_manager.allocReg(null, abi.RegisterClass.sse), + abi_size, + ) else null, + }; + const mask_lock = + if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null; + defer if (mask_lock) |lock| self.register_manager.unlockReg(lock); + + const ordered_air: [2]Air.Inst.Ref = if (lhs_ty.isVector(zcu) and + switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .bool => false, + .int => switch (air_tag) { + .cmp_lt, .cmp_gte => true, + else => false, + }, + .float => switch (air_tag) { + .cmp_gte, .cmp_gt => true, + else => false, + }, + else => unreachable, + }) .{ rhs_air, lhs_air } else .{ lhs_air, rhs_air }; + + if (lhs_ty.isAbiInt(zcu)) for (ordered_air) |op_air| { + switch (try self.resolveInst(op_air)) { + .register => |op_reg| switch (op_reg.class()) { + .sse => try self.register_manager.getReg(op_reg, null), + else => {}, + }, + else => {}, + } + }; + + const lhs_mcv = try self.resolveInst(ordered_air[0]); + var rhs_mcv = try self.resolveInst(ordered_air[1]); + switch (lhs_mcv) { + .immediate => |imm| switch (imm) { + 0 => switch (air_tag) { + .sub, .sub_wrap => return self.genUnOp(maybe_inst, .neg, ordered_air[1]), + else => {}, + }, + else => {}, + }, + else => {}, + } + + const is_commutative = switch (air_tag) { + .add, + .add_wrap, + .mul, + .bool_or, + .bit_or, + .bool_and, + .bit_and, + .xor, + .min, + .max, + .cmp_eq, + .cmp_neq, + => true, + + else => false, + }; + + const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) { + .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null }, + .register_pair => |lhs_regs| locks: { + const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs); + break :locks .{ locks[0], locks[1] }; + }, + else => @splat(null), + }; + defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) { + .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null }, + .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs), + else => @splat(null), + }; + defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + var flipped = false; + var copied_to_dst = true; + const dst_mcv: MCValue = dst: { + const tracked_inst = switch (air_tag) { + else => maybe_inst, + .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null, + }; + if (maybe_inst) |inst| { + if ((!sse_op or lhs_mcv.isRegister()) and + self.reuseOperandAdvanced(inst, ordered_air[0], 0, lhs_mcv, tracked_inst)) + break :dst lhs_mcv; + if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and + self.reuseOperandAdvanced(inst, ordered_air[1], 1, rhs_mcv, tracked_inst)) + { + flipped = true; + break :dst rhs_mcv; + } + } + const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true); + if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx)) + copied_to_dst = false + else + try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{}); + rhs_mcv = try self.resolveInst(ordered_air[1]); + break :dst dst_mcv; + }; + const dst_locks: [2]?RegisterLock = switch (dst_mcv) { + .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null }, + .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs), + else => @splat(null), + }; + defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const unmat_src_mcv = if (flipped) lhs_mcv else rhs_mcv; + const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg| + if (self.hasFeature(.avx) and unmat_src_mcv.isRegister() and maybe_inst != null and + self.liveness.operandDies(maybe_inst.?, if (flipped) 0 else 1)) unmat_src_mcv else src: { + try self.genSetReg(mask_reg, rhs_ty, unmat_src_mcv, .{}); + break :src .{ .register = mask_reg }; + } + else + unmat_src_mcv; + const src_locks: [2]?RegisterLock = switch (src_mcv) { + .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null }, + .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs), + else => @splat(null), + }; + defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock); + + if (!sse_op) { + switch (air_tag) { + .add, + .add_wrap, + => try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv), + + .sub, + .sub_wrap, + => try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv), + + .ptr_add, + .ptr_sub, + => { + const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv); + const tmp_mcv = MCValue{ .register = tmp_reg }; + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const elem_size = lhs_ty.elemType2(zcu).abiSize(zcu); + try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size }); + try self.genBinOpMir( + switch (air_tag) { + .ptr_add => .{ ._, .add }, + .ptr_sub => .{ ._, .sub }, + else => unreachable, + }, + lhs_ty, + dst_mcv, + tmp_mcv, + ); + }, + + .bool_or, + .bit_or, + => try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv), + + .bool_and, + .bit_and, + => try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv), + + .xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv), + + .min, + .max, + => { + const resolved_src_mcv = switch (src_mcv) { + else => src_mcv, + .air_ref => |src_ref| try self.resolveInst(src_ref), + }; + + if (abi_size > 8) { + const dst_regs = switch (dst_mcv) { + .register_pair => |dst_regs| dst_regs, + else => dst: { + const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); + const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs); + defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock); + + try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{}); + break :dst dst_regs; + }, + }; + const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs); + defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock| + self.register_manager.unlockReg(lock); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const signed = lhs_ty.isSignedInt(zcu); + const cc: Condition = switch (air_tag) { + .min => if (signed) .nl else .nb, + .max => if (signed) .nge else .nae, + else => unreachable, + }; + + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); + if (src_mcv.isBase()) { + try self.asmRegisterMemory( + .{ ._, .cmp }, + dst_regs[0], + try src_mcv.mem(self, .{ .size = .qword }), + ); + try self.asmRegisterMemory( + .{ ._, .sbb }, + tmp_reg, + try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ); + try self.asmCmovccRegisterMemory( + cc, + dst_regs[0], + try src_mcv.mem(self, .{ .size = .qword }), + ); + try self.asmCmovccRegisterMemory( + cc, + dst_regs[1], + try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), + ); + } else { + try self.asmRegisterRegister( + .{ ._, .cmp }, + dst_regs[0], + src_mcv.register_pair[0], + ); + try self.asmRegisterRegister( + .{ ._, .sbb }, + tmp_reg, + src_mcv.register_pair[1], + ); + try self.asmCmovccRegisterRegister(cc, dst_regs[0], src_mcv.register_pair[0]); + try self.asmCmovccRegisterRegister(cc, dst_regs[1], src_mcv.register_pair[1]); + } + try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs }, .{}); + } else { + const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) { + .immediate, + .eflags, + .register_offset, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => true, + .memory => |addr| std.math.cast(i32, @as(i64, @bitCast(addr))) == null, + else => false, + .register_pair, + .register_overflow, + => unreachable, + }) + .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) } + else + resolved_src_mcv; + const mat_mcv_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv); + + const int_info = lhs_ty.intInfo(zcu); + const cc: Condition = switch (int_info.signedness) { + .unsigned => switch (air_tag) { + .min => .a, + .max => .b, + else => unreachable, + }, + .signed => switch (air_tag) { + .min => .g, + .max => .l, + else => unreachable, + }, + }; + + const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(zcu))), 2); + const tmp_reg = switch (dst_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(lhs_ty, dst_mcv), + }; + const tmp_lock = self.register_manager.lockReg(tmp_reg); + defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); + switch (mat_src_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .register_overflow, + .register_mask, + .indirect_load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .register => |src_reg| try self.asmCmovccRegisterRegister( + cc, + registerAlias(tmp_reg, cmov_abi_size), + registerAlias(src_reg, cmov_abi_size), + ), + .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( + cc, + registerAlias(tmp_reg, cmov_abi_size), + switch (mat_src_mcv) { + .memory => |addr| .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = .fromSize(cmov_abi_size), + .disp = @intCast(@as(i64, @bitCast(addr))), + } }, + }, + .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ .rm = .{ + .size = .fromSize(cmov_abi_size), + .disp = reg_off.off, + } }, + }, + .load_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .fromSize(cmov_abi_size), + .disp = frame_addr.off, + } }, + }, + else => unreachable, + }, + ), + } + try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg }, .{}); + } + }, + + .cmp_eq, .cmp_neq => { + assert(lhs_ty.isVector(zcu) and lhs_ty.childType(zcu).toIntern() == .bool_type); + try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv); + switch (air_tag) { + .cmp_eq => try self.genUnOpMir(.{ ._, .not }, lhs_ty, dst_mcv), + .cmp_neq => {}, + else => unreachable, + } + }, + + else => return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + } + return dst_mcv; + } + + const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size); + const mir_tag = @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { + else => unreachable, + .float => switch (lhs_ty.floatBits(self.target)) { + 16 => { + assert(self.hasFeature(.f16c)); + const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); + + const tmp_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_w, .insr }, + dst_reg, + lhs_reg, + try src_mcv.mem(self, .{ .size = .word }), + .u(1), + ) else try self.asmRegisterRegisterRegister( + .{ .vp_, .unpcklwd }, + dst_reg, + lhs_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); + try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); + try self.asmRegisterRegisterRegister( + switch (air_tag) { + .add => .{ .v_ss, .add }, + .sub => .{ .v_ss, .sub }, + .mul => .{ .v_ss, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, + .max => .{ .v_ss, .max }, + .min => .{ .v_ss, .min }, + else => unreachable, + }, + dst_reg, + dst_reg, + tmp_reg, + ); + switch (air_tag) { + .div_trunc, .div_floor => try self.asmRegisterRegisterRegisterImmediate( + .{ .v_ss, .round }, + dst_reg, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{ + .direction = switch (air_tag) { + .div_trunc => .zero, + .div_floor => .down, + else => unreachable, + }, + .precision = .inexact, + }), + ), + else => {}, + } + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + return dst_mcv; + }, + 32 => switch (air_tag) { + .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, + .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, + .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul }, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div }, + .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, + .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, + else => unreachable, + }, + 64 => switch (air_tag) { + .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, + .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, + .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul }, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div }, + .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, + .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, + else => unreachable, + }, + 80, 128 => null, + else => unreachable, + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + else => null, + .int => switch (lhs_ty.childType(zcu).intInfo(zcu).bits) { + 8 => switch (lhs_ty.vectorLen(zcu)) { + 1...16 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx)) .{ .vp_b, .add } else .{ .p_b, .add }, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx)) .{ .vp_b, .sub } else .{ .p_b, .sub }, + .bit_and => if (self.hasFeature(.avx)) + .{ .vp_, .@"and" } + else + .{ .p_, .@"and" }, + .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, + .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, + .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_b, .mins } + else if (self.hasFeature(.sse4_1)) + .{ .p_b, .mins } + else + null, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_b, .minu } + else if (self.hasFeature(.sse4_1)) + .{ .p_b, .minu } + else + null, + }, + .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_b, .maxs } + else if (self.hasFeature(.sse4_1)) + .{ .p_b, .maxs } + else + null, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_b, .maxu } + else if (self.hasFeature(.sse4_1)) + .{ .p_b, .maxu } + else + null, + }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_b, .cmpgt } + else + .{ .p_b, .cmpgt }, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_b, .cmpeq } else .{ .p_b, .cmpeq }, + else => null, + }, + 17...32 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_b, .add } else null, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_b, .sub } else null, + .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, + .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, + .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, + .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_b, .mins } else null, + .unsigned => if (self.hasFeature(.avx)) .{ .vp_b, .minu } else null, + }, + .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_b, .maxs } else null, + .unsigned => if (self.hasFeature(.avx2)) .{ .vp_b, .maxu } else null, + }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) .{ .vp_b, .cmpgt } else null, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_b, .cmpeq } else null, + else => null, + }, + else => null, + }, + 16 => switch (lhs_ty.vectorLen(zcu)) { + 1...8 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx)) .{ .vp_w, .add } else .{ .p_w, .add }, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx)) .{ .vp_w, .sub } else .{ .p_w, .sub }, + .mul, + .mul_wrap, + => if (self.hasFeature(.avx)) .{ .vp_w, .mull } else .{ .p_d, .mull }, + .bit_and => if (self.hasFeature(.avx)) + .{ .vp_, .@"and" } + else + .{ .p_, .@"and" }, + .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, + .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, + .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_w, .mins } + else + .{ .p_w, .mins }, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_w, .minu } + else + .{ .p_w, .minu }, + }, + .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_w, .maxs } + else + .{ .p_w, .maxs }, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_w, .maxu } + else + .{ .p_w, .maxu }, + }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_w, .cmpgt } + else + .{ .p_w, .cmpgt }, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_w, .cmpeq } else .{ .p_w, .cmpeq }, + else => null, + }, + 9...16 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_w, .add } else null, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_w, .sub } else null, + .mul, + .mul_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_w, .mull } else null, + .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, + .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, + .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, + .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .mins } else null, + .unsigned => if (self.hasFeature(.avx)) .{ .vp_w, .minu } else null, + }, + .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .maxs } else null, + .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .maxu } else null, + }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) .{ .vp_w, .cmpgt } else null, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_w, .cmpeq } else null, + else => null, + }, + else => null, + }, + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1...4 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx)) .{ .vp_d, .add } else .{ .p_d, .add }, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx)) .{ .vp_d, .sub } else .{ .p_d, .sub }, + .mul, + .mul_wrap, + => if (self.hasFeature(.avx)) + .{ .vp_d, .mull } + else if (self.hasFeature(.sse4_1)) + .{ .p_d, .mull } + else + null, + .bit_and => if (self.hasFeature(.avx)) + .{ .vp_, .@"and" } + else + .{ .p_, .@"and" }, + .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, + .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, + .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_d, .mins } + else if (self.hasFeature(.sse4_1)) + .{ .p_d, .mins } + else + null, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_d, .minu } + else if (self.hasFeature(.sse4_1)) + .{ .p_d, .minu } + else + null, + }, + .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_d, .maxs } + else if (self.hasFeature(.sse4_1)) + .{ .p_d, .maxs } + else + null, + .unsigned => if (self.hasFeature(.avx)) + .{ .vp_d, .maxu } + else if (self.hasFeature(.sse4_1)) + .{ .p_d, .maxu } + else + null, + }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_d, .cmpgt } + else + .{ .p_d, .cmpgt }, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else .{ .p_d, .cmpeq }, + else => null, + }, + 5...8 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_d, .add } else null, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_d, .sub } else null, + .mul, + .mul_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_d, .mull } else null, + .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, + .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, + .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, + .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .mins } else null, + .unsigned => if (self.hasFeature(.avx)) .{ .vp_d, .minu } else null, + }, + .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .maxs } else null, + .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .maxu } else null, + }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) .{ .vp_d, .cmpgt } else null, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else null, + else => null, + }, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1...2 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx)) .{ .vp_q, .add } else .{ .p_q, .add }, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx)) .{ .vp_q, .sub } else .{ .p_q, .sub }, + .bit_and => if (self.hasFeature(.avx)) + .{ .vp_, .@"and" } + else + .{ .p_, .@"and" }, + .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" }, + .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor }, + .cmp_lt, + .cmp_lte, + .cmp_gte, + .cmp_gt, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) + .{ .vp_q, .cmpgt } + else if (self.hasFeature(.sse4_2)) + .{ .p_q, .cmpgt } + else + null, + .unsigned => null, + }, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) + .{ .vp_q, .cmpeq } + else if (self.hasFeature(.sse4_1)) + .{ .p_q, .cmpeq } + else + null, + else => null, + }, + 3...4 => switch (air_tag) { + .add, + .add_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_q, .add } else null, + .sub, + .sub_wrap, + => if (self.hasFeature(.avx2)) .{ .vp_q, .sub } else null, + .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null, + .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null, + .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null, + .cmp_eq, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else null, + .cmp_lt, + .cmp_lte, + .cmp_gt, + .cmp_gte, + => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) { + .signed => if (self.hasFeature(.avx)) .{ .vp_d, .cmpgt } else null, + .unsigned => null, + }, + else => null, + }, + else => null, + }, + else => null, + }, + .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { + 16 => tag: { + assert(self.hasFeature(.f16c)); + const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); + switch (lhs_ty.vectorLen(zcu)) { + 1 => { + const tmp_reg = + (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_w, .insr }, + dst_reg, + lhs_reg, + try src_mcv.mem(self, .{ .size = .word }), + .u(1), + ) else try self.asmRegisterRegisterRegister( + .{ .vp_, .unpcklwd }, + dst_reg, + lhs_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); + try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); + try self.asmRegisterRegisterRegister( + switch (air_tag) { + .add => .{ .v_ss, .add }, + .sub => .{ .v_ss, .sub }, + .mul => .{ .v_ss, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, + .max => .{ .v_ss, .max }, + .min => .{ .v_ss, .max }, + else => unreachable, + }, + dst_reg, + dst_reg, + tmp_reg, + ); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + return dst_mcv; + }, + 2 => { + const tmp_reg = (try self.register_manager.allocReg( + null, + abi.RegisterClass.sse, + )).to128(); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_d, .insr }, + dst_reg, + lhs_reg, + try src_mcv.mem(self, .{ .size = .dword }), + .u(1), + ) else try self.asmRegisterRegisterRegister( + .{ .v_ps, .unpckl }, + dst_reg, + lhs_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); + try self.asmRegisterRegisterRegister( + .{ .v_ps, .movhl }, + tmp_reg, + dst_reg, + dst_reg, + ); + try self.asmRegisterRegisterRegister( + switch (air_tag) { + .add => .{ .v_ps, .add }, + .sub => .{ .v_ps, .sub }, + .mul => .{ .v_ps, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, + .max => .{ .v_ps, .max }, + .min => .{ .v_ps, .max }, + else => unreachable, + }, + dst_reg, + dst_reg, + tmp_reg, + ); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + return dst_mcv; + }, + 3...4 => { + const tmp_reg = (try self.register_manager.allocReg( + null, + abi.RegisterClass.sse, + )).to128(); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, lhs_reg); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_ps, .cvtph2 }, + tmp_reg, + try src_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ .v_ps, .cvtph2 }, + tmp_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + try self.asmRegisterRegisterRegister( + switch (air_tag) { + .add => .{ .v_ps, .add }, + .sub => .{ .v_ps, .sub }, + .mul => .{ .v_ps, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, + .max => .{ .v_ps, .max }, + .min => .{ .v_ps, .max }, + else => unreachable, + }, + dst_reg, + dst_reg, + tmp_reg, + ); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg, + bits.RoundMode.imm(.{}), + ); + return dst_mcv; + }, + 5...8 => { + const tmp_reg = (try self.register_manager.allocReg( + null, + abi.RegisterClass.sse, + )).to256(); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), lhs_reg); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_ps, .cvtph2 }, + tmp_reg, + try src_mcv.mem(self, .{ .size = .xword }), + ) else try self.asmRegisterRegister( + .{ .v_ps, .cvtph2 }, + tmp_reg, + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), + ); + try self.asmRegisterRegisterRegister( + switch (air_tag) { + .add => .{ .v_ps, .add }, + .sub => .{ .v_ps, .sub }, + .mul => .{ .v_ps, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, + .max => .{ .v_ps, .max }, + .min => .{ .v_ps, .max }, + else => unreachable, + }, + dst_reg.to256(), + dst_reg.to256(), + tmp_reg, + ); + try self.asmRegisterRegisterImmediate( + .{ .v_, .cvtps2ph }, + dst_reg, + dst_reg.to256(), + bits.RoundMode.imm(.{}), + ); + return dst_mcv; + }, + else => break :tag null, + } + }, + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1 => switch (air_tag) { + .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, + .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, + .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul }, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div }, + .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, + .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, + .cmp_lt, + .cmp_lte, + .cmp_eq, + .cmp_gte, + .cmp_gt, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp }, + else => unreachable, + }, + 2...4 => switch (air_tag) { + .add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add }, + .sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub }, + .mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul }, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div }, + .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max }, + .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min }, + .cmp_lt, + .cmp_lte, + .cmp_eq, + .cmp_gte, + .cmp_gt, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp }, + else => unreachable, + }, + 5...8 => if (self.hasFeature(.avx)) switch (air_tag) { + .add => .{ .v_ps, .add }, + .sub => .{ .v_ps, .sub }, + .mul => .{ .v_ps, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, + .max => .{ .v_ps, .max }, + .min => .{ .v_ps, .min }, + .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp }, + else => unreachable, + } else null, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1 => switch (air_tag) { + .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, + .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, + .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul }, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div }, + .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, + .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, + .cmp_lt, + .cmp_lte, + .cmp_eq, + .cmp_gte, + .cmp_gt, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp }, + else => unreachable, + }, + 2 => switch (air_tag) { + .add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add }, + .sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub }, + .mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul }, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div }, + .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max }, + .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min }, + .cmp_lt, + .cmp_lte, + .cmp_eq, + .cmp_gte, + .cmp_gt, + .cmp_neq, + => if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp }, + else => unreachable, + }, + 3...4 => if (self.hasFeature(.avx)) switch (air_tag) { + .add => .{ .v_pd, .add }, + .sub => .{ .v_pd, .sub }, + .mul => .{ .v_pd, .mul }, + .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div }, + .max => .{ .v_pd, .max }, + .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp }, + .min => .{ .v_pd, .min }, + else => unreachable, + } else null, + else => null, + }, + 80, 128 => null, + else => unreachable, + }, + }, + }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }); + + const lhs_copy_reg = if (maybe_mask_reg) |_| registerAlias( + if (copied_to_dst) try self.copyToTmpRegister(lhs_ty, dst_mcv) else lhs_mcv.getReg().?, + abi_size, + ) else null; + const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null; + defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock); + + switch (mir_tag[1]) { + else => if (self.hasFeature(.avx)) { + const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); + if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( + mir_tag, + dst_reg, + lhs_reg, + try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { + else => .fromSize(abi_size), + .vector => dst_reg.size(), + } }), + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_reg, + lhs_reg, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), + ); + } else { + assert(copied_to_dst); + if (src_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + dst_reg, + try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { + else => .fromSize(abi_size), + .vector => dst_reg.size(), + } }), + ) else try self.asmRegisterRegister( + mir_tag, + dst_reg, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), + ); + }, + .cmp => { + const imm: Immediate = .u(switch (air_tag) { + .cmp_eq => 0, + .cmp_lt, .cmp_gt => 1, + .cmp_lte, .cmp_gte => 2, + .cmp_neq => 4, + else => unreachable, + }); + if (self.hasFeature(.avx)) { + const lhs_reg = + if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); + if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + mir_tag, + dst_reg, + lhs_reg, + try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { + else => .fromSize(abi_size), + .vector => dst_reg.size(), + } }), + imm, + ) else try self.asmRegisterRegisterRegisterImmediate( + mir_tag, + dst_reg, + lhs_reg, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), + imm, + ); + } else { + assert(copied_to_dst); + if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( + mir_tag, + dst_reg, + try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { + else => .fromSize(abi_size), + .vector => dst_reg.size(), + } }), + imm, + ) else try self.asmRegisterRegisterImmediate( + mir_tag, + dst_reg, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size), + imm, + ); + } + }, + } + + switch (air_tag) { + .bit_and, .bit_or, .xor => {}, + .max, .min => if (maybe_mask_reg) |mask_reg| if (self.hasFeature(.avx)) { + const rhs_copy_reg = registerAlias(src_mcv.getReg().?, abi_size); + + try self.asmRegisterRegisterRegisterImmediate( + @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { + .float => switch (lhs_ty.floatBits(self.target)) { + 32 => .{ .v_ss, .cmp }, + 64 => .{ .v_sd, .cmp }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1 => .{ .v_ss, .cmp }, + 2...8 => .{ .v_ps, .cmp }, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1 => .{ .v_sd, .cmp }, + 2...4 => .{ .v_pd, .cmp }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + mask_reg, + rhs_copy_reg, + rhs_copy_reg, + bits.VexFloatPredicate.imm(.unord), + ); + try self.asmRegisterRegisterRegisterRegister( + @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { + .float => switch (lhs_ty.floatBits(self.target)) { + 32 => .{ .v_ps, .blendv }, + 64 => .{ .v_pd, .blendv }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1...8 => .{ .v_ps, .blendv }, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1...4 => .{ .v_pd, .blendv }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + dst_reg, + dst_reg, + lhs_copy_reg.?, + mask_reg, + ); + } else { + const has_blend = self.hasFeature(.sse4_1); + try self.asmRegisterRegisterImmediate( + @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { + .float => switch (lhs_ty.floatBits(self.target)) { + 32 => .{ ._ss, .cmp }, + 64 => .{ ._sd, .cmp }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1 => .{ ._ss, .cmp }, + 2...4 => .{ ._ps, .cmp }, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1 => .{ ._sd, .cmp }, + 2 => .{ ._pd, .cmp }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + mask_reg, + mask_reg, + bits.SseFloatPredicate.imm(if (has_blend) .unord else .ord), + ); + if (has_blend) try self.asmRegisterRegisterRegister( + @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { + .float => switch (lhs_ty.floatBits(self.target)) { + 32 => .{ ._ps, .blendv }, + 64 => .{ ._pd, .blendv }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1...4 => .{ ._ps, .blendv }, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1...2 => .{ ._pd, .blendv }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }), + dst_reg, + lhs_copy_reg.?, + mask_reg, + ) else { + const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(zcu)) { + .float => switch (lhs_ty.floatBits(self.target)) { + 32 => ._ps, + 64 => ._pd, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) { + 32 => switch (lhs_ty.vectorLen(zcu)) { + 1...4 => ._ps, + else => null, + }, + 64 => switch (lhs_ty.vectorLen(zcu)) { + 1...2 => ._pd, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{ + @tagName(air_tag), lhs_ty.fmt(pt), + }); + try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg); + try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?); + try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg); + } + }, + .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => { + switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) { + .int => switch (air_tag) { + .cmp_lt, + .cmp_eq, + .cmp_gt, + => {}, + .cmp_lte, + .cmp_gte, + .cmp_neq, + => { + const unsigned_ty = try lhs_ty.toUnsigned(pt); + const not_mcv = try self.lowerValue(try unsigned_ty.maxInt(pt, unsigned_ty)); + const not_mem: Memory = if (not_mcv.isBase()) + try not_mcv.mem(self, .{ .size = .fromSize(abi_size) }) + else + .{ .base = .{ + .reg = try self.copyToTmpRegister(.usize, not_mcv.address()), + }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } }; + switch (mir_tag[0]) { + .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory( + .{ .vp_, .xor }, + dst_reg, + dst_reg, + not_mem, + ), + .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory( + .{ .p_, .xor }, + dst_reg, + not_mem, + ), + else => unreachable, + } + }, + else => unreachable, + }, + .float => {}, + else => unreachable, + } + + const gp_reg = try self.register_manager.allocReg(maybe_inst, abi.RegisterClass.gp); + const gp_lock = self.register_manager.lockRegAssumeUnused(gp_reg); + defer self.register_manager.unlockReg(gp_lock); + + try self.asmRegisterRegister(switch (mir_tag[0]) { + ._pd, ._sd, .p_q => .{ ._pd, .movmsk }, + ._ps, ._ss, .p_d => .{ ._ps, .movmsk }, + .p_b => .{ .p_b, .movmsk }, + .p_w => movmsk: { + try self.asmRegisterRegister(.{ .p_b, .ackssw }, dst_reg, dst_reg); + break :movmsk .{ .p_b, .movmsk }; + }, + .v_pd, .v_sd, .vp_q => .{ .v_pd, .movmsk }, + .v_ps, .v_ss, .vp_d => .{ .v_ps, .movmsk }, + .vp_b => .{ .vp_b, .movmsk }, + .vp_w => movmsk: { + try self.asmRegisterRegisterRegister( + .{ .vp_b, .ackssw }, + dst_reg, + dst_reg, + dst_reg, + ); + break :movmsk .{ .vp_b, .movmsk }; + }, + else => unreachable, + }, gp_reg.to32(), dst_reg); + return .{ .register = gp_reg }; + }, + else => unreachable, + } + + return dst_mcv; +} + +fn genBinOpMir( + self: *CodeGen, + mir_tag: Mir.Inst.FixedTag, + ty: Type, + dst_mcv: MCValue, + src_mcv: MCValue, +) !void { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + try self.spillEflagsIfOccupied(); + switch (dst_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register_overflow, + .register_mask, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // unmodifiable destination + .register, .register_pair, .register_triple, .register_quadruple, .register_offset => { + switch (dst_mcv) { + .register, .register_pair, .register_triple, .register_quadruple => {}, + .register_offset => |ro| assert(ro.off == 0), + else => unreachable, + } + for (dst_mcv.getRegs(), 0..) |dst_reg, dst_reg_i| { + const dst_reg_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); + + const mir_limb_tag: Mir.Inst.FixedTag = switch (dst_reg_i) { + 0 => mir_tag, + 1 => switch (mir_tag[1]) { + .add => .{ ._, .adc }, + .sub, .cmp => .{ ._, .sbb }, + .@"or", .@"and", .xor => mir_tag, + else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{ + @tagName(mir_tag[1]), + }), + }, + else => unreachable, + }; + const off: u4 = @intCast(dst_reg_i * 8); + const limb_abi_size = @min(abi_size - off, 8); + const dst_alias = registerAlias(dst_reg, limb_abi_size); + switch (src_mcv) { + .none, + .unreach, + .dead, + .undef, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + => unreachable, + .register, + .register_pair, + .register_triple, + .register_quadruple, + => try self.asmRegisterRegister( + mir_limb_tag, + dst_alias, + registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size), + ), + .immediate => |imm| { + assert(off == 0); + switch (self.regBitSize(ty)) { + 8 => try self.asmRegisterImmediate( + mir_limb_tag, + dst_alias, + if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small| + .s(small) + else + .u(@as(u8, @intCast(imm))), + ), + 16 => try self.asmRegisterImmediate( + mir_limb_tag, + dst_alias, + if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small| + .s(small) + else + .u(@as(u16, @intCast(imm))), + ), + 32 => try self.asmRegisterImmediate( + mir_limb_tag, + dst_alias, + if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + .s(small) + else + .u(@as(u32, @intCast(imm))), + ), + 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small)) + else + try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias( + try self.copyToTmpRegister(ty, src_mcv), + limb_abi_size, + )), + else => unreachable, + } + }, + .eflags, + .register_offset, + .memory, + .indirect, + .load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => { + direct: { + try self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) { + .memory => |addr| .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = .fromSize(limb_abi_size), + .disp = std.math.cast(i32, addr + off) orelse break :direct, + } }, + }, + .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ .rm = .{ + .size = .fromSize(limb_abi_size), + .disp = reg_off.off + off, + } }, + }, + .load_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .fromSize(limb_abi_size), + .disp = frame_addr.off + off, + } }, + }, + else => break :direct, + }); + continue; + } + + switch (src_mcv) { + .eflags, + .register_offset, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => { + assert(off == 0); + const reg = try self.copyToTmpRegister(ty, src_mcv); + return self.genBinOpMir( + mir_limb_tag, + ty, + dst_mcv, + .{ .register = reg }, + ); + }, + .memory, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => { + const ptr_ty = try pt.singleConstPtrType(ty); + const addr_reg = try self.copyToTmpRegister(ptr_ty, src_mcv.address()); + return self.genBinOpMir(mir_limb_tag, ty, dst_mcv, .{ + .indirect = .{ .reg = addr_reg, .off = off }, + }); + }, + else => unreachable, + } + }, + .air_ref => |src_ref| return self.genBinOpMir( + mir_tag, + ty, + dst_mcv, + try self.resolveInst(src_ref), + ), + } + } + }, + .memory, + .indirect, + .load_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => { + const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock }; + const limb_abi_size: u32 = @min(abi_size, 8); + + const dst_info: OpInfo = switch (dst_mcv) { + else => unreachable, + .memory, .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => dst: { + const dst_addr_reg = + (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); + const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); + errdefer self.register_manager.unlockReg(dst_addr_lock); + + try self.genSetReg(dst_addr_reg, .usize, dst_mcv.address(), .{}); + break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock }; + }, + .load_frame => null, + }; + defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock); + + const resolved_src_mcv = switch (src_mcv) { + else => src_mcv, + .air_ref => |src_ref| try self.resolveInst(src_ref), + }; + const src_info: OpInfo = switch (resolved_src_mcv) { + .none, + .unreach, + .dead, + .undef, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .immediate, + .eflags, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .indirect, + .load_frame, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => null, + .memory, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => src: { + switch (resolved_src_mcv) { + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and + std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null) + break :src null, + .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, + else => unreachable, + } + + const src_addr_reg = + (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); + const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); + errdefer self.register_manager.unlockReg(src_addr_lock); + + try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{}); + break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; + }, + }; + defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); + + const ty_signedness = + if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned; + const limb_ty: Type = if (abi_size <= 8) ty else switch (ty_signedness) { + .signed => .usize, + .unsigned => .isize, + }; + var limb_i: usize = 0; + var off: i32 = 0; + while (off < abi_size) : ({ + limb_i += 1; + off += 8; + }) { + const mir_limb_tag: Mir.Inst.FixedTag = switch (limb_i) { + 0 => mir_tag, + else => switch (mir_tag[1]) { + .add => .{ ._, .adc }, + .sub, .cmp => .{ ._, .sbb }, + .@"or", .@"and", .xor => mir_tag, + else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{ + @tagName(mir_tag[1]), + }), + }, + }; + const dst_limb_mem: Memory = switch (dst_mcv) { + .memory, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => .{ + .base = .{ .reg = dst_info.?.addr_reg }, + .mod = .{ .rm = .{ + .size = .fromSize(limb_abi_size), + .disp = off, + } }, + }, + .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ .rm = .{ + .size = .fromSize(limb_abi_size), + .disp = reg_off.off + off, + } }, + }, + .load_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .fromSize(limb_abi_size), + .disp = frame_addr.off + off, + } }, + }, + else => unreachable, + }; + switch (resolved_src_mcv) { + .none, + .unreach, + .dead, + .undef, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .immediate => |src_imm| { + const imm: u64 = switch (limb_i) { + 0 => src_imm, + else => switch (ty_signedness) { + .signed => @bitCast(@as(i64, @bitCast(src_imm)) >> 63), + .unsigned => 0, + }, + }; + switch (self.regBitSize(limb_ty)) { + 8 => try self.asmMemoryImmediate( + mir_limb_tag, + dst_limb_mem, + if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small| + .s(small) + else + .u(@as(u8, @intCast(imm))), + ), + 16 => try self.asmMemoryImmediate( + mir_limb_tag, + dst_limb_mem, + if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small| + .s(small) + else + .u(@as(u16, @intCast(imm))), + ), + 32 => try self.asmMemoryImmediate( + mir_limb_tag, + dst_limb_mem, + if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + .s(small) + else + .u(@as(u32, @intCast(imm))), + ), + 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small)) + else + try self.asmMemoryRegister( + mir_limb_tag, + dst_limb_mem, + registerAlias( + try self.copyToTmpRegister(limb_ty, .{ .immediate = imm }), + limb_abi_size, + ), + ), + else => unreachable, + } + }, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .eflags, + .memory, + .indirect, + .load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => { + const src_limb_mcv: MCValue = if (src_info) |info| .{ + .indirect = .{ .reg = info.addr_reg, .off = off }, + } else switch (resolved_src_mcv) { + .register, .register_pair, .register_triple, .register_quadruple => .{ + .register = resolved_src_mcv.getRegs()[limb_i], + }, + .eflags, + .register_offset, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => switch (limb_i) { + 0 => resolved_src_mcv, + else => .{ .immediate = 0 }, + }, + .memory => |addr| .{ .memory = @bitCast(@as(i64, @bitCast(addr)) + off) }, + .indirect => |reg_off| .{ .indirect = .{ + .reg = reg_off.reg, + .off = reg_off.off + off, + } }, + .load_frame => |frame_addr| .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }, + else => unreachable, + }; + const src_limb_reg = if (src_limb_mcv.isRegister()) + src_limb_mcv.getReg().? + else + try self.copyToTmpRegister(limb_ty, src_limb_mcv); + try self.asmMemoryRegister( + mir_limb_tag, + dst_limb_mem, + registerAlias(src_limb_reg, limb_abi_size), + ); + }, + } + } + }, + } +} + +/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv. +/// Does not support byte-size operands. +fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void { + const pt = self.pt; + const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu)); + try self.spillEflagsIfOccupied(); + switch (dst_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register_offset, + .register_overflow, + .register_mask, + .indirect_load_frame, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // unmodifiable destination + .register => |dst_reg| { + const alias_size = switch (abi_size) { + 1 => 4, + else => abi_size, + }; + const dst_alias = registerAlias(dst_reg, alias_size); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + switch (abi_size) { + 1 => try self.asmRegisterRegister(.{ ._, .movzx }, dst_reg.to32(), dst_reg.to8()), + else => {}, + } + + const resolved_src_mcv = switch (src_mcv) { + else => src_mcv, + .air_ref => |src_ref| try self.resolveInst(src_ref), + }; + switch (resolved_src_mcv) { + .none, + .unreach, + .dead, + .undef, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .register => |src_reg| { + switch (abi_size) { + 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()), + else => {}, + } + try self.asmRegisterRegister( + .{ .i_, .mul }, + dst_alias, + registerAlias(src_reg, alias_size), + ); + }, + .immediate => |imm| { + if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| { + try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst_alias, dst_alias, .s(small)); + } else { + const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); + return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg }); + } + }, + .register_offset, + .eflags, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => { + const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); + switch (abi_size) { + 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()), + else => {}, + } + try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size)); + }, + .memory, .indirect, .load_frame => switch (abi_size) { + 1 => { + const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); + try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()); + try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size)); + }, + else => try self.asmRegisterMemory( + .{ .i_, .mul }, + dst_alias, + switch (resolved_src_mcv) { + .memory => |addr| .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse + return self.asmRegisterRegister( + .{ .i_, .mul }, + dst_alias, + registerAlias( + try self.copyToTmpRegister(dst_ty, resolved_src_mcv), + abi_size, + ), + ), + } }, + }, + .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = reg_off.off, + } }, + }, + .load_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = frame_addr.off, + } }, + }, + else => unreachable, + }, + ), + }, + } + }, + .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented + .memory, + .indirect, + .load_frame, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => { + const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); + const tmp_mcv = MCValue{ .register = tmp_reg }; + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv); + try self.genCopy(dst_ty, dst_mcv, tmp_mcv, .{}); + }, + } +} + +fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const arg_index = for (self.args, 0..) |arg, arg_index| { + if (arg != .none) break arg_index; + } else unreachable; + const src_mcv = self.args[arg_index]; + self.args = self.args[arg_index + 1 ..]; + const result: MCValue = if (self.mod.strip and self.liveness.isUnused(inst)) .unreach else result: { + const arg_ty = self.typeOfIndex(inst); + switch (src_mcv) { + .register, .register_pair, .load_frame => { + for (src_mcv.getRegs()) |reg| self.register_manager.getRegAssumeFree(reg, inst); + break :result src_mcv; + }, + .indirect => |reg_off| { + self.register_manager.getRegAssumeFree(reg_off.reg, null); + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.genCopy(arg_ty, dst_mcv, src_mcv, .{}); + break :result dst_mcv; + }, + .indirect_load_frame => |frame_addr| { + const dst_mcv = try self.allocRegOrMem(inst, false); + const ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); + defer self.register_manager.unlockReg(ptr_lock); + try self.genSetReg(ptr_reg, .usize, .{ .load_frame = frame_addr }, .{}); + try self.genCopy(arg_ty, dst_mcv, .{ .indirect = .{ .reg = ptr_reg } }, .{}); + break :result dst_mcv; + }, + .elementwise_args => |regs_frame_addr| { + try self.spillEflagsIfOccupied(); + + const fn_info = zcu.typeToFunc(self.fn_type).?; + const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc); + var prev_reg: Register = undefined; + for ( + param_int_regs[param_int_regs.len - regs_frame_addr.regs ..], + 0.., + ) |dst_reg, elem_index| { + assert(self.register_manager.isRegFree(dst_reg)); + if (elem_index > 0) { + try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index)); + try self.asmRegisterRegister( + .{ ._, .@"or" }, + dst_reg.to8(), + prev_reg.to8(), + ); + } + prev_reg = dst_reg; + } + + const prev_lock = if (regs_frame_addr.regs > 0) + self.register_manager.lockRegAssumeUnused(prev_reg) + else + null; + defer if (prev_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = try self.allocRegOrMem(inst, false); + if (regs_frame_addr.regs > 0) try self.asmMemoryRegister( + .{ ._, .mov }, + try dst_mcv.mem(self, .{ .size = .byte }), + prev_reg.to8(), + ); + try self.genInlineMemset( + dst_mcv.address().offset(@intFromBool(regs_frame_addr.regs > 0)), + .{ .immediate = 0 }, + .{ .immediate = arg_ty.abiSize(zcu) - @intFromBool(regs_frame_addr.regs > 0) }, + .{}, + ); + + const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); + defer self.register_manager.unlockReg(index_lock); + + try self.asmRegisterImmediate( + .{ ._, .mov }, + index_reg.to32(), + .u(regs_frame_addr.regs), + ); + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmMemoryImmediate(.{ ._, .cmp }, .{ + .base = .{ .frame = regs_frame_addr.frame_index }, + .mod = .{ .rm = .{ + .size = .byte, + .index = index_reg.to64(), + .scale = .@"8", + .disp = regs_frame_addr.frame_off - @as(u6, regs_frame_addr.regs) * 8, + } }, + }, Immediate.u(0)); + const unset = try self.asmJccReloc(.e, undefined); + try self.asmMemoryRegister( + .{ ._s, .bt }, + try dst_mcv.mem(self, .{ .size = .dword }), + index_reg.to32(), + ); + self.performReloc(unset); + if (self.hasFeature(.slow_incdec)) { + try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, index_reg.to32()); + } + try self.asmRegisterImmediate( + .{ ._, .cmp }, + index_reg.to32(), + .u(arg_ty.vectorLen(zcu)), + ); + _ = try self.asmJccReloc(.b, loop); + + break :result dst_mcv; + }, + else => return self.fail("TODO implement arg for {f}", .{src_mcv}), + } + }; + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn genLocalDebugInfo(cg: *CodeGen, air_tag: Air.Inst.Tag, ty: Type, mcv: MCValue) !void { + assert(!cg.mod.strip); + _ = switch (air_tag) { + else => unreachable, + .arg, .dbg_var_val, .dbg_arg_inline => switch (mcv) { + .none, .unreach, .dead, .elementwise_args, .reserved_frame, .air_ref => unreachable, + .immediate => |imm| if (std.math.cast(u32, imm)) |small| try cg.addInst(.{ + .tag = .pseudo, + .ops = switch (air_tag) { + else => unreachable, + .arg, .dbg_arg_inline => .pseudo_dbg_arg_i_u, + .dbg_var_val => .pseudo_dbg_var_i_u, + }, + .data = .{ .i = .{ .i = small } }, + }) else try cg.addInst(.{ + .tag = .pseudo, + .ops = switch (air_tag) { + else => unreachable, + .arg, .dbg_arg_inline => .pseudo_dbg_arg_i_64, + .dbg_var_val => .pseudo_dbg_var_i_64, + }, + .data = .{ .i64 = imm }, + }), + .lea_frame => |frame_addr| try cg.addInst(.{ + .tag = .pseudo, + .ops = switch (air_tag) { + else => unreachable, + .arg, .dbg_arg_inline => .pseudo_dbg_arg_fa, + .dbg_var_val => .pseudo_dbg_var_fa, + }, + .data = .{ .fa = frame_addr }, + }), + else => { + const frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu)); + try cg.genSetMem(.{ .frame = frame_index }, 0, ty, mcv, .{}); + _ = try cg.addInst(.{ + .tag = .pseudo, + .ops = switch (air_tag) { + else => unreachable, + .arg, .dbg_arg_inline => .pseudo_dbg_arg_m, + .dbg_var_val => .pseudo_dbg_var_m, + }, + .data = .{ .x = .{ + .payload = try cg.addExtra(Mir.Memory.encode(.{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .qword } }, + })), + } }, + }); + }, + }, + .dbg_var_ptr => switch (mcv) { + else => unreachable, + .none, .unreach, .dead, .elementwise_args, .reserved_frame, .air_ref => unreachable, + .lea_frame => |frame_addr| try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_var_m, + .data = .{ .x = .{ + .payload = try cg.addExtra(Mir.Memory.encode(.{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = frame_addr.off, + } }, + })), + } }, + }), + .lea_nav => |nav| try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_var_m, + .data = .{ .x = .{ + .payload = try cg.addExtra(Mir.Memory.encode(.{ + .base = .{ .nav = nav }, + .mod = .{ .rm = .{ .size = .qword } }, + })), + } }, + }), + .lea_uav => |uav| try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_var_m, + .data = .{ .x = .{ + .payload = try cg.addExtra(Mir.Memory.encode(.{ + .base = .{ .uav = uav }, + .mod = .{ .rm = .{ .size = .qword } }, + })), + } }, + }), + .lea_lazy_sym => |lazy_sym| try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_var_m, + .data = .{ .x = .{ + .payload = try cg.addExtra(Mir.Memory.encode(.{ + .base = .{ .lazy_sym = lazy_sym }, + .mod = .{ .rm = .{ .size = .qword } }, + })), + } }, + }), + .lea_extern_func => |extern_func| try cg.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_var_m, + .data = .{ .x = .{ + .payload = try cg.addExtra(Mir.Memory.encode(.{ + .base = .{ .extern_func = extern_func }, + .mod = .{ .rm = .{ .size = .qword } }, + })), + } }, + }), + }, + }; +} + +fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier, opts: CopyOptions) !void { + if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{}); + + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Call, pl_op.payload); + const arg_refs: []const Air.Inst.Ref = + @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); + + const ExpectedContents = extern struct { + tys: [32][@sizeOf(Type)]u8 align(@alignOf(Type)), + vals: [32][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)), + }; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); + const allocator = stack.get(); + + const arg_tys = try allocator.alloc(Type, arg_refs.len); + defer allocator.free(arg_tys); + for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = self.typeOf(arg_ref); + + const arg_vals = try allocator.alloc(MCValue, arg_refs.len); + defer allocator.free(arg_vals); + for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref }; + + const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals, opts); + + var bt = self.liveness.iterateBigTomb(inst); + try self.feed(&bt, pl_op.operand); + for (arg_refs) |arg_ref| try self.feed(&bt, arg_ref); + + const result = if (self.liveness.isUnused(inst)) .unreach else ret; + return self.finishAirResult(inst, result); +} + +fn genCall(self: *CodeGen, info: union(enum) { + air: Air.Inst.Ref, + extern_func: struct { + return_type: InternPool.Index, + param_types: []const InternPool.Index, + sym: []const u8, + }, +}, arg_types: []const Type, args: []const MCValue, opts: CopyOptions) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const fn_ty = switch (info) { + .air => |callee| fn_info: { + const callee_ty = self.typeOf(callee); + break :fn_info switch (callee_ty.zigTypeTag(zcu)) { + .@"fn" => callee_ty, + .pointer => callee_ty.childType(zcu), + else => unreachable, + }; + }, + .extern_func => |extern_func| try pt.funcType(.{ + .param_types = extern_func.param_types, + .return_type = extern_func.return_type, + .cc = self.target.cCallingConvention().?, + }), + }; + const fn_info = zcu.typeToFunc(fn_ty).?; + + const ExpectedContents = extern struct { + var_args: [32][@sizeOf(Type)]u8 align(@alignOf(Type)), + frame_indices: [32]FrameIndex, + reg_locks: [32][@sizeOf(?RegisterLock)]u8 align(@alignOf(?RegisterLock)), + }; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); + const allocator = stack.get(); + + const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len); + defer allocator.free(var_args); + for (var_args, arg_types[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; + + const frame_indices = try allocator.alloc(FrameIndex, args.len); + defer allocator.free(frame_indices); + + var reg_locks: std.array_list.Managed(?RegisterLock) = .init(allocator); + defer reg_locks.deinit(); + try reg_locks.ensureTotalCapacity(16); + defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); + + var call_info = try self.resolveCallingConventionValues(fn_info, var_args, .call_frame); + defer call_info.deinit(self); + + // We need a properly aligned and sized call frame to be able to call this function. + { + const needed_call_frame: FrameAlloc = .init(.{ + .size = call_info.stack_byte_count, + .alignment = call_info.stack_align, + }); + const frame_allocs_slice = self.frame_allocs.slice(); + const stack_frame_size = + &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size); + const stack_frame_align = + &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align); + } + + try self.spillEflagsIfOccupied(); + try self.spillCallerPreservedRegs(fn_info.cc, call_info.err_ret_trace_reg); + + // set stack arguments first because this can clobber registers + // also clobber spill arguments as we go + switch (call_info.return_value.long) { + .none, .unreach => {}, + .indirect => |reg_off| try self.register_manager.getReg(reg_off.reg, null), + else => unreachable, + } + for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index| + switch (dst_arg) { + .none => {}, + .register => |reg| { + try self.register_manager.getReg(reg, null); + try reg_locks.append(self.register_manager.lockReg(reg)); + }, + .register_pair => |regs| { + for (regs) |reg| try self.register_manager.getReg(reg, null); + try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs)); + }, + .indirect => |reg_off| { + frame_index.* = try self.allocFrameIndex(.initType(arg_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); + try self.register_manager.getReg(reg_off.reg, null); + try reg_locks.append(self.register_manager.lockReg(reg_off.reg)); + }, + .load_frame => { + try self.genCopy(arg_ty, dst_arg, src_arg, opts); + try self.freeValue(src_arg, .{}); + }, + .elementwise_args => |regs_frame_addr| { + const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); + defer self.register_manager.unlockReg(index_lock); + + const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ + .base = .{ .reg = try self.copyToTmpRegister(.usize, switch (src_arg) { + else => src_arg, + .air_ref => |src_ref| try self.resolveInst(src_ref), + }.address()) }, + .mod = .{ .rm = .{ .size = .dword } }, + }; + const src_lock = switch (src_mem.base) { + .reg => |src_reg| self.register_manager.lockReg(src_reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + try self.asmRegisterImmediate( + .{ ._, .mov }, + index_reg.to32(), + .u(regs_frame_addr.regs), + ); + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32()); + try self.asmSetccMemory(.c, .{ + .base = .{ .frame = regs_frame_addr.frame_index }, + .mod = .{ .rm = .{ + .size = .byte, + .index = index_reg.to64(), + .scale = .@"8", + .disp = regs_frame_addr.frame_off - @as(u6, regs_frame_addr.regs) * 8, + } }, + }); + if (self.hasFeature(.slow_incdec)) { + try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); + } else { + try self.asmRegister(.{ ._c, .in }, index_reg.to32()); + } + try self.asmRegisterImmediate( + .{ ._, .cmp }, + index_reg.to32(), + .u(arg_ty.vectorLen(zcu)), + ); + _ = try self.asmJccReloc(.b, loop); + + const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc); + for (param_int_regs[param_int_regs.len - regs_frame_addr.regs ..]) |dst_reg| { + try self.register_manager.getReg(dst_reg, null); + try reg_locks.append(self.register_manager.lockReg(dst_reg)); + } + }, + else => unreachable, + }; + + if (call_info.err_ret_trace_reg != .none) { + if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { + if (switch (err_ret_trace.short) { + .register => |reg| call_info.err_ret_trace_reg != reg, + else => true, + }) { + try self.register_manager.getReg(call_info.err_ret_trace_reg, err_ret_trace_index); + try reg_locks.append(self.register_manager.lockReg(call_info.err_ret_trace_reg)); + + try self.genSetReg(call_info.err_ret_trace_reg, .usize, err_ret_trace.short, .{}); + err_ret_trace.trackMaterialize(err_ret_trace_index, .{ + .long = err_ret_trace.long, + .short = .{ .register = call_info.err_ret_trace_reg }, + }); + } + } + } + + // now we are free to set register arguments + switch (call_info.return_value.long) { + .none, .unreach => {}, + .indirect => |reg_off| { + const ret_ty: Type = .fromInterned(fn_info.return_type); + const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu)); + try self.genSetReg(reg_off.reg, .usize, .{ + .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, + }, .{}); + call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } }; + try reg_locks.append(self.register_manager.lockReg(reg_off.reg)); + }, + else => unreachable, + } + + for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| + switch (dst_arg) { + .none, .load_frame => {}, + .register => |dst_reg| switch (fn_info.cc) { + else => try self.genSetReg(registerAlias( + dst_reg, + @intCast(arg_ty.abiSize(zcu)), + ), arg_ty, src_arg, opts), + .x86_64_sysv, .x86_64_win => { + const promoted_ty = self.promoteInt(arg_ty); + const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(zcu)); + const dst_alias = registerAlias(dst_reg, promoted_abi_size); + try self.genSetReg(dst_alias, promoted_ty, src_arg, opts); + if (promoted_ty.toIntern() != arg_ty.toIntern()) + try self.truncateRegister(arg_ty, dst_alias); + }, + }, + .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, opts), + .indirect => |reg_off| try self.genSetReg(reg_off.reg, .usize, .{ + .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, + }, .{}), + .elementwise_args => |regs_frame_addr| { + const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ + .base = .{ .reg = try self.copyToTmpRegister( + .usize, + switch (src_arg) { + else => src_arg, + .air_ref => |src_ref| try self.resolveInst(src_ref), + }.address(), + ) }, + .mod = .{ .rm = .{ .size = .dword } }, + }; + const src_lock = switch (src_mem.base) { + .reg => |src_reg| self.register_manager.lockReg(src_reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const param_int_regs = abi.getCAbiIntParamRegs(fn_info.cc); + for ( + param_int_regs[param_int_regs.len - regs_frame_addr.regs ..], + 0.., + ) |dst_reg, elem_index| { + try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); + try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index)); + try self.asmSetccRegister(.c, dst_reg.to8()); + } + }, + else => unreachable, + }; + + if (fn_info.is_var_args) try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count)); + + // Due to incremental compilation, how function calls are generated depends + // on linking. + switch (info) { + .air => |callee| if (try self.air.value(callee, pt)) |func_value| { + const func_key = ip.indexToKey(func_value.ip_index); + switch (switch (func_key) { + else => func_key, + .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { + .nav => |nav| ip.indexToKey(zcu.navValue(nav).toIntern()), + else => func_key, + } else func_key, + }) { + else => unreachable, + .func => |func| try self.asmImmediate(.{ ._, .call }, .{ .nav = .{ .index = func.owner_nav } }), + .@"extern" => |@"extern"| try self.asmImmediate(.{ ._, .call }, .{ .nav = .{ .index = @"extern".owner_nav } }), + } + } else { + assert(self.typeOf(callee).zigTypeTag(zcu) == .pointer); + const scratch_reg = abi.getCAbiLinkerScratchReg(fn_info.cc); + try self.genSetReg(scratch_reg, .usize, .{ .air_ref = callee }, .{}); + try self.asmRegister(.{ ._, .call }, scratch_reg); + }, + .extern_func => |extern_func| try self.asmImmediate(.{ ._, .call }, .{ .extern_func = try self.addString(extern_func.sym) }), + } + return call_info.return_value.short; +} + +fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { + const pt = self.pt; + const zcu = pt.zcu; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + + const ret_ty = self.fn_type.fnReturnType(zcu); + switch (self.ret_mcv.short) { + .none => {}, + .register => |reg| { + const reg_lock = self.register_manager.lockRegAssumeUnused(reg); + defer self.register_manager.unlockReg(reg_lock); + try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); + }, + inline .register_pair, .register_triple, .register_quadruple => |regs| { + const reg_locks = self.register_manager.lockRegsAssumeUnused(regs.len, regs); + defer for (reg_locks) |reg_lock| self.register_manager.unlockReg(reg_lock); + try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); + }, + .indirect => |reg_off| { + try self.register_manager.getReg(reg_off.reg, null); + const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg); + defer self.register_manager.unlockReg(lock); + + try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{}); + try self.genSetMem( + .{ .reg = reg_off.reg }, + reg_off.off, + ret_ty, + .{ .air_ref = un_op }, + .{ .safety = safety }, + ); + }, + else => unreachable, + } + self.ret_mcv.liveOut(self, inst); + + if (self.err_ret_trace_reg != .none) { + if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { + if (switch (err_ret_trace.short) { + .register => |reg| self.err_ret_trace_reg != reg, + else => true, + }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{}); + err_ret_trace.liveOut(self, err_ret_trace_index); + } + } + + try self.finishAir(inst, .unreach, .{ un_op, .none, .none }); + + // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction + // which is available if the jump is 127 bytes or less forward. + const jmp_reloc = try self.asmJmpReloc(undefined); + try self.epilogue_relocs.append(self.gpa, jmp_reloc); +} + +fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ptr = try self.resolveInst(un_op); + + const ptr_ty = self.typeOf(un_op); + switch (self.ret_mcv.short) { + .none => {}, + .register, .register_pair => try self.load(self.ret_mcv.short, ptr_ty, ptr), + .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr, .{}), + else => unreachable, + } + self.ret_mcv.liveOut(self, inst); + + if (self.err_ret_trace_reg != .none) { + if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { + if (switch (err_ret_trace.short) { + .register => |reg| self.err_ret_trace_reg != reg, + else => true, + }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{}); + err_ret_trace.liveOut(self, err_ret_trace_index); + } + } + + try self.finishAir(inst, .unreach, .{ un_op, .none, .none }); + + // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction + // which is available if the jump is 127 bytes or less forward. + const jmp_reloc = try self.asmJmpReloc(undefined); + try self.epilogue_relocs.append(self.gpa, jmp_reloc); +} + +fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + var ty = self.typeOf(bin_op.lhs); + var null_compare: ?Mir.Inst.Index = null; + + const result: Condition = result: { + try self.spillEflagsIfOccupied(); + + const lhs_mcv = try self.resolveInst(bin_op.lhs); + const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) { + .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null }, + .register_pair => |lhs_regs| locks: { + const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs); + break :locks .{ locks[0], locks[1] }; + }, + .register_offset => |lhs_ro| .{ + self.register_manager.lockRegAssumeUnused(lhs_ro.reg), + null, + }, + else => @splat(null), + }; + defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(bin_op.rhs); + const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) { + .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null }, + .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs), + .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null }, + else => @splat(null), + }; + defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + switch (ty.zigTypeTag(zcu)) { + .float => { + const float_bits = ty.floatBits(self.target); + if (!switch (float_bits) { + 16 => self.hasFeature(.f16c), + 32 => self.hasFeature(.sse), + 64 => self.hasFeature(.sse2), + 80, 128 => false, + else => unreachable, + }) { + var sym_buf: ["__???f2".len]u8 = undefined; + const ret = try self.genCall(.{ .extern_func = .{ + .return_type = .i32_type, + .param_types = &.{ ty.toIntern(), ty.toIntern() }, + .sym = std.fmt.bufPrint(&sym_buf, "__{s}{c}f2", .{ + switch (op) { + .eq => "eq", + .neq => "ne", + .lt => "lt", + .lte => "le", + .gt => "gt", + .gte => "ge", + }, + floatCompilerRtAbiName(float_bits), + }) catch unreachable, + } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, .{}); + try self.genBinOpMir(.{ ._, .@"test" }, .i32, ret, ret); + break :result switch (op) { + .eq => .e, + .neq => .ne, + .lt => .l, + .lte => .le, + .gt => .g, + .gte => .ge, + }; + } + }, + .optional => if (!ty.optionalReprIsPayload(zcu)) { + const opt_ty = ty; + const opt_abi_size: u31 = @intCast(opt_ty.abiSize(zcu)); + ty = opt_ty.optionalChild(zcu); + const payload_abi_size: u31 = @intCast(ty.abiSize(zcu)); + + const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg); + defer self.register_manager.unlockReg(temp_lhs_lock); + + if (lhs_mcv.isBase()) try self.asmRegisterMemory( + .{ ._, .mov }, + temp_lhs_reg.to8(), + try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }), + ) else { + try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{}); + try self.asmRegisterImmediate( + .{ ._r, .sh }, + registerAlias(temp_lhs_reg, opt_abi_size), + .u(payload_abi_size * 8), + ); + } + + const payload_compare = payload_compare: { + if (rhs_mcv.isBase()) { + const rhs_mem = + try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }); + try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8()); + const payload_compare = try self.asmJccReloc(.nz, undefined); + try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem); + break :payload_compare payload_compare; + } + + const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv); + const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg); + defer self.register_manager.unlockReg(temp_rhs_lock); + + try self.asmRegisterImmediate( + .{ ._r, .sh }, + registerAlias(temp_rhs_reg, opt_abi_size), + .u(payload_abi_size * 8), + ); + try self.asmRegisterRegister( + .{ ._, .@"test" }, + temp_lhs_reg.to8(), + temp_rhs_reg.to8(), + ); + const payload_compare = try self.asmJccReloc(.nz, undefined); + try self.asmRegisterRegister( + .{ ._, .cmp }, + temp_lhs_reg.to8(), + temp_rhs_reg.to8(), + ); + break :payload_compare payload_compare; + }; + null_compare = try self.asmJmpReloc(undefined); + self.performReloc(payload_compare); + }, + else => {}, + } + + switch (ty.zigTypeTag(zcu)) { + else => { + const abi_size: u16 = @intCast(ty.abiSize(zcu)); + const may_flip: enum { + may_flip, + must_flip, + must_not_flip, + } = if (abi_size > 8) switch (op) { + .lt, .gte => .must_not_flip, + .lte, .gt => .must_flip, + .eq, .neq => .may_flip, + } else .may_flip; + + const flipped = switch (may_flip) { + .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(), + .must_flip => true, + .must_not_flip => false, + }; + const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv; + const dst_mcv = if (unmat_dst_mcv.isRegister() or + (abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: { + const dst_mcv = try self.allocTempRegOrMem(ty, true); + try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{}); + break :dst dst_mcv; + }; + const dst_lock = + if (dst_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const src_mcv = try self.resolveInst(if (flipped) bin_op.lhs else bin_op.rhs); + const src_lock = + if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + break :result .fromCompareOperator( + if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned, + result_op: { + const flipped_op = if (flipped) op.reverse() else op; + if (abi_size > 8) switch (flipped_op) { + .lt, .gte => {}, + .lte, .gt => unreachable, + .eq, .neq => { + const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock }; + + const resolved_dst_mcv = switch (dst_mcv) { + else => dst_mcv, + .air_ref => |dst_ref| try self.resolveInst(dst_ref), + }; + const dst_info: OpInfo = switch (resolved_dst_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register_offset, + .register_overflow, + .register_mask, + .indirect, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .load_frame, + => null, + .memory, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => dst: { + switch (resolved_dst_mcv) { + .memory => |addr| if (std.math.cast( + i32, + @as(i64, @bitCast(addr)), + ) != null and std.math.cast( + i32, + @as(i64, @bitCast(addr)) + abi_size - 8, + ) != null) break :dst null, + .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, + else => unreachable, + } + + const dst_addr_reg = (try self.register_manager.allocReg( + null, + abi.RegisterClass.gp, + )).to64(); + const dst_addr_lock = + self.register_manager.lockRegAssumeUnused(dst_addr_reg); + errdefer self.register_manager.unlockReg(dst_addr_lock); + + try self.genSetReg(dst_addr_reg, .usize, resolved_dst_mcv.address(), .{}); + break :dst .{ + .addr_reg = dst_addr_reg, + .addr_lock = dst_addr_lock, + }; + }, + }; + defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock); + + const resolved_src_mcv = switch (src_mcv) { + else => src_mcv, + .air_ref => |src_ref| try self.resolveInst(src_ref), + }; + const src_info: OpInfo = switch (resolved_src_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register, + .register_offset, + .register_overflow, + .register_mask, + .indirect, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .register_pair, + .register_triple, + .register_quadruple, + .load_frame, + => null, + .memory, + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => src: { + switch (resolved_src_mcv) { + .memory => |addr| if (std.math.cast( + i32, + @as(i64, @bitCast(addr)), + ) != null and std.math.cast( + i32, + @as(i64, @bitCast(addr)) + abi_size - 8, + ) != null) break :src null, + .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, + else => unreachable, + } + + const src_addr_reg = (try self.register_manager.allocReg( + null, + abi.RegisterClass.gp, + )).to64(); + const src_addr_lock = + self.register_manager.lockRegAssumeUnused(src_addr_reg); + errdefer self.register_manager.unlockReg(src_addr_lock); + + try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{}); + break :src .{ + .addr_reg = src_addr_reg, + .addr_lock = src_addr_lock, + }; + }, + }; + defer if (src_info) |info| + self.register_manager.unlockReg(info.addr_lock); + + const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); + const acc_reg = regs[0].to64(); + const locks = self.register_manager.lockRegsAssumeUnused(2, regs); + defer for (locks) |lock| self.register_manager.unlockReg(lock); + + const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable; + var limb_i: u16 = 0; + while (limb_i < limbs_len) : (limb_i += 1) { + const off = limb_i * 8; + const tmp_reg = regs[@min(limb_i, 1)].to64(); + + try self.genSetReg(tmp_reg, .usize, if (dst_info) |info| .{ + .indirect = .{ .reg = info.addr_reg, .off = off }, + } else switch (resolved_dst_mcv) { + inline .register_pair, + .register_triple, + .register_quadruple, + => |dst_regs| .{ .register = dst_regs[limb_i] }, + .memory => |dst_addr| .{ + .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off), + }, + .indirect => |reg_off| .{ .indirect = .{ + .reg = reg_off.reg, + .off = reg_off.off + off, + } }, + .load_frame => |frame_addr| .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }, + else => unreachable, + }, .{}); + + try self.genBinOpMir( + .{ ._, .xor }, + .usize, + .{ .register = tmp_reg }, + if (src_info) |info| .{ + .indirect = .{ .reg = info.addr_reg, .off = off }, + } else switch (resolved_src_mcv) { + inline .register_pair, + .register_triple, + .register_quadruple, + => |src_regs| .{ .register = src_regs[limb_i] }, + .memory => |src_addr| .{ + .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off), + }, + .indirect => |reg_off| .{ .indirect = .{ + .reg = reg_off.reg, + .off = reg_off.off + off, + } }, + .load_frame => |frame_addr| .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }, + else => unreachable, + }, + ); + + if (limb_i > 0) + try self.asmRegisterRegister(.{ ._, .@"or" }, acc_reg, tmp_reg); + } + assert(limbs_len >= 2); // use flags from or + break :result_op flipped_op; + }, + }; + try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv); + break :result_op flipped_op; + }, + ); + }, + .float => { + const flipped = switch (op) { + .lt, .lte => true, + .eq, .gte, .gt, .neq => false, + }; + + const dst_mcv = if (flipped) rhs_mcv else lhs_mcv; + const dst_reg = if (dst_mcv.isRegister()) + dst_mcv.getReg().? + else + try self.copyToTmpRegister(ty, dst_mcv); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + const src_mcv = if (flipped) lhs_mcv else rhs_mcv; + + switch (ty.floatBits(self.target)) { + 16 => { + assert(self.hasFeature(.f16c)); + const tmp1_reg = + (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); + const tmp1_mcv = MCValue{ .register = tmp1_reg }; + const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg); + defer self.register_manager.unlockReg(tmp1_lock); + + const tmp2_reg = + (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128(); + const tmp2_mcv = MCValue{ .register = tmp2_reg }; + const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg); + defer self.register_manager.unlockReg(tmp2_lock); + + if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_w, .insr }, + tmp1_reg, + dst_reg.to128(), + try src_mcv.mem(self, .{ .size = .word }), + .u(1), + ) else try self.asmRegisterRegisterRegister( + .{ .vp_, .unpcklwd }, + tmp1_reg, + dst_reg.to128(), + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(ty, src_mcv)).to128(), + ); + try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, tmp1_reg, tmp1_reg); + try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg); + try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv); + }, + 32 => try self.genBinOpMir( + .{ ._ss, .ucomi }, + ty, + .{ .register = dst_reg }, + src_mcv, + ), + 64 => try self.genBinOpMir( + .{ ._sd, .ucomi }, + ty, + .{ .register = dst_reg }, + src_mcv, + ), + else => unreachable, + } + + break :result switch (if (flipped) op.reverse() else op) { + .lt, .lte => unreachable, // required to have been canonicalized to gt(e) + .gt => .a, + .gte => .ae, + .eq => .z_and_np, + .neq => .nz_or_p, + }; + }, + } + }; + + if (null_compare) |reloc| self.performReloc(reloc); + self.eflags_inst = inst; + return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; + const dst_mcv = try self.genBinOp( + inst, + .fromCmpOp(extra.compareOperator(), false), + extra.lhs, + extra.rhs, + ); + return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); +} + +fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Try, pl_op.payload); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); + const operand_ty = self.typeOf(pl_op.operand); + const result = try self.genTry(inst, pl_op.operand, body, operand_ty, false); + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); + const operand_ty = self.typeOf(extra.data.ptr); + const result = try self.genTry(inst, extra.data.ptr, body, operand_ty, true); + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn genTry( + self: *CodeGen, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, + body: []const Air.Inst.Index, + operand_ty: Type, + operand_is_ptr: bool, +) !MCValue { + const liveness_cond_br = self.liveness.getCondBr(inst); + + const operand_mcv = try self.resolveInst(operand); + const is_err_mcv = if (operand_is_ptr) + try self.isErrPtr(null, operand_ty, operand_mcv) + else + try self.isErr(null, operand_ty, operand_mcv); + + const reloc = try self.genCondBrMir(.anyerror, is_err_mcv); + + if (self.liveness.operandDies(inst, 0)) { + if (operand.toIndex()) |operand_inst| try self.processDeath(operand_inst, .{}); + } + + const state = try self.saveState(); + + for (liveness_cond_br.else_deaths) |death| try self.processDeath(death, .{}); + try self.genBodyBlock(body); + try self.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + self.performReloc(reloc); + + for (liveness_cond_br.then_deaths) |death| try self.processDeath(death, .{}); + + const result = if (self.liveness.isUnused(inst)) + .unreach + else if (operand_is_ptr) + try self.genUnwrapErrUnionPayloadPtrMir(inst, operand_ty, operand_mcv) + else + try self.genUnwrapErrUnionPayloadMir(inst, operand_ty, operand_mcv); + return result; +} + +fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index { + const pt = self.pt; + const abi_size = ty.abiSize(pt.zcu); + switch (mcv) { + .eflags => |cc| { + // Here we map the opposites since the jump is to the false branch. + return self.asmJccReloc(cc.negate(), undefined); + }, + .register => |reg| { + try self.spillEflagsIfOccupied(); + try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1)); + return self.asmJccReloc(.z, undefined); + }, + .immediate, + .load_frame, + => { + try self.spillEflagsIfOccupied(); + if (abi_size <= 8) { + const reg = try self.copyToTmpRegister(ty, mcv); + return self.genCondBrMir(ty, .{ .register = reg }); + } + return self.fail("TODO implement condbr when condition is {f} with abi larger than 8 bytes", .{mcv}); + }, + else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}), + } +} + +fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const cond = try self.resolveInst(pl_op.operand); + const cond_ty = self.typeOf(pl_op.operand); + const extra = self.air.extraData(Air.CondBr, pl_op.payload); + const then_body: []const Air.Inst.Index = + @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = + @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const liveness_cond_br = self.liveness.getCondBr(inst); + + // If the condition dies here in this condbr instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + if (self.liveness.operandDies(inst, 0)) { + if (pl_op.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); + } + + const state = try self.saveState(); + const reloc = try self.genCondBrMir(cond_ty, cond); + + for (liveness_cond_br.then_deaths) |death| try self.processDeath(death, .{}); + try self.genBodyBlock(then_body); + try self.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + self.performReloc(reloc); + + for (liveness_cond_br.else_deaths) |death| try self.processDeath(death, .{}); + try self.genBodyBlock(else_body); + try self.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + // We already took care of pl_op.operand earlier, so there's nothing left to do. +} + +fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const err_ty = eu_ty.errorUnionSet(zcu); + if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false + + try self.spillEflagsIfOccupied(); + + const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); + switch (eu_mcv) { + .register => |reg| { + const eu_lock = self.register_manager.lockReg(reg); + defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); + + const tmp_reg = try self.copyToTmpRegister(eu_ty, eu_mcv); + if (err_off > 0) { + try self.genShiftBinOpMir( + .{ ._r, .sh }, + eu_ty, + .{ .register = tmp_reg }, + .u8, + .{ .immediate = @as(u6, @intCast(err_off * 8)) }, + ); + } else { + try self.truncateRegister(.anyerror, tmp_reg); + } + try self.genBinOpMir(.{ ._, .cmp }, .anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 }); + }, + .load_frame => |frame_addr| try self.genBinOpMir( + .{ ._, .cmp }, + .anyerror, + .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + err_off, + } }, + .{ .immediate = 0 }, + ), + else => return self.fail("TODO implement isErr for {f}", .{eu_mcv}), + } + + if (maybe_inst) |inst| self.eflags_inst = inst; + return MCValue{ .eflags = .a }; +} + +fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const eu_ty = ptr_ty.childType(zcu); + const err_ty = eu_ty.errorUnionSet(zcu); + if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false + + try self.spillEflagsIfOccupied(); + + const ptr_reg = switch (ptr_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(ptr_ty, ptr_mcv), + }; + const ptr_lock = self.register_manager.lockReg(ptr_reg); + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); + try self.asmMemoryImmediate( + .{ ._, .cmp }, + .{ + .base = .{ .reg = ptr_reg }, + .mod = .{ .rm = .{ + .size = self.memSize(.anyerror), + .disp = err_off, + } }, + }, + .u(0), + ); + + if (maybe_inst) |inst| self.eflags_inst = inst; + return MCValue{ .eflags = .a }; +} + +fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { + // A block is a setup to be able to jump to the end. + const inst_tracking_i = self.inst_tracking.count(); + self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach)); + + try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() }); + const liveness = self.liveness.getBlock(inst); + + try self.genBody(body); + + var block_data = self.blocks.fetchRemove(inst).?; + defer block_data.value.deinit(self.gpa); + if (block_data.value.relocs.items.len > 0) { + var last_inst: Mir.Inst.Index = @intCast(self.mir_instructions.len - 1); + while (block_data.value.relocs.getLastOrNull() == last_inst) { + block_data.value.relocs.items.len -= 1; + self.mir_instructions.set(last_inst, .{ + .tag = .pseudo, + .ops = .pseudo_dead_none, + .data = undefined, + }); + last_inst -= 1; + } + for (block_data.value.relocs.items) |block_reloc| self.performReloc(block_reloc); + try self.restoreState(block_data.value.state, liveness.deaths, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + } + + if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i); + const tracking = &self.inst_tracking.values()[inst_tracking_i]; + if (self.liveness.isUnused(inst)) try tracking.die(self, inst, .{}); + self.getValueIfFree(tracking.short, inst); +} + +fn lowerSwitchBr( + cg: *CodeGen, + inst: Air.Inst.Index, + switch_br: Air.UnwrappedSwitch, + condition: MCValue, + condition_dies: bool, + is_loop: bool, +) !void { + const zcu = cg.pt.zcu; + const condition_ty = cg.typeOf(switch_br.operand); + const unsigned_condition_ty = try cg.pt.intType(.unsigned, cg.intInfo(condition_ty).?.bits); + + const ExpectedContents = extern struct { + liveness_deaths: [1 << 8 | 1]Air.Inst.Index, + bigint_limbs: [std.math.big.int.calcTwosCompLimbCount(1 << 10)]std.math.big.Limb, + relocs: [1 << 6]Mir.Inst.Index, + }; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); + const allocator = stack.get(); + + const state = try cg.saveState(); + + const liveness = try cg.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1); + defer allocator.free(liveness.deaths); + + if (!cg.mod.pic and cg.target.ofmt == .elf) table: { + var prong_items: u32 = 0; + var min: ?Value = null; + var max: ?Value = null; + { + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + prong_items += @intCast(case.items.len + case.ranges.len); + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val; + if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val; + } + for (case.ranges) |range| { + const low = Value.fromInterned(range[0].toInterned().?); + if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low; + const high = Value.fromInterned(range[1].toInterned().?); + if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high; + } + } + } + // This condition also triggers for switches with no non-else prongs and switches on bool. + if (prong_items < 1 << 2 or prong_items > 1 << 8) break :table; + + var min_space: Value.BigIntSpace = undefined; + const min_bigint = min.?.toBigInt(&min_space, zcu); + var max_space: Value.BigIntSpace = undefined; + const max_bigint = max.?.toBigInt(&max_space, zcu); + const limbs = try allocator.alloc( + std.math.big.Limb, + @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1, + ); + defer allocator.free(limbs); + const table_len = table_len: { + var table_len_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + table_len_bigint.sub(max_bigint, min_bigint); + assert(table_len_bigint.positive); // min <= max + break :table_len @as(u11, table_len_bigint.toConst().toInt(u10) catch break :table) + 1; // no more than a 1024 entry table + }; + assert(prong_items <= table_len); // each prong item introduces at least one unique integer to the range + if (prong_items < table_len >> 2) break :table; // no more than 75% waste + + const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: { + const condition_index = try cg.allocTempRegOrMem(condition_ty, true); + try cg.genCopy(condition_ty, condition_index, condition, .{}); + break :condition_index condition_index; + }; + try cg.spillEflagsIfOccupied(); + if (min.?.orderAgainstZero(zcu).compare(.neq)) try cg.genBinOpMir( + .{ ._, .sub }, + condition_ty, + condition_index, + .{ .air_ref = Air.internedToRef(min.?.toIntern()) }, + ); + const else_reloc = if (switch_br.else_body_len > 0) else_reloc: { + var cond_temp = try cg.tempInit(unsigned_condition_ty, condition_index); + var table_max_temp = try cg.tempFromValue(try cg.pt.intValue(unsigned_condition_ty, table_len - 1)); + const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, cg) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try cond_temp.die(cg); + try table_max_temp.die(cg); + const else_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); + try cc_temp.die(cg); + break :else_reloc else_reloc; + } else undefined; + const table_start: u31 = @intCast(cg.mir_table.items.len); + { + const condition_index_reg = condition_index_reg: { + if (condition_index.isRegister()) { + const condition_index_reg = condition_index.getReg().?; + if (condition_index_reg.isClass(.general_purpose)) break :condition_index_reg condition_index_reg; + } + break :condition_index_reg try cg.copyToTmpRegister(.usize, condition_index); + }; + const condition_index_lock = cg.register_manager.lockReg(condition_index_reg); + defer if (condition_index_lock) |lock| cg.register_manager.unlockReg(lock); + try cg.truncateRegister(condition_ty, condition_index_reg); + const ptr_size = @divExact(cg.target.ptrBitWidth(), 8); + try cg.asmMemory(.{ ._mp, .j }, .{ + .base = .table, + .mod = .{ .rm = .{ + .size = .ptr, + .index = condition_index_reg.toSize(.ptr, cg.target), + .scale = .fromFactor(@intCast(ptr_size)), + .disp = table_start * ptr_size, + } }, + }); + } + const else_reloc_marker: u32 = 0; + assert(cg.mir_instructions.len > else_reloc_marker); + try cg.mir_table.appendNTimes(cg.gpa, else_reloc_marker, table_len); + if (is_loop) try cg.loop_switches.putNoClobber(cg.gpa, inst, .{ + .start = table_start, + .len = table_len, + .min = min.?, + .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable", + }); + defer if (is_loop) { + var loop_switch_data = cg.loop_switches.fetchRemove(inst).?.value; + switch (loop_switch_data.else_relocs) { + .@"unreachable", .backward => {}, + .forward => |*else_relocs| else_relocs.deinit(cg.gpa), + } + }; + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + { + const table = cg.mir_table.items[table_start..][0..table_len]; + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + var val_space: Value.BigIntSpace = undefined; + const val_bigint = val.toBigInt(&val_space, zcu); + var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + index_bigint.sub(val_bigint, min_bigint); + table[index_bigint.toConst().toInt(u10) catch unreachable] = @intCast(cg.mir_instructions.len); + } + for (case.ranges) |range| { + var low_space: Value.BigIntSpace = undefined; + const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu); + var high_space: Value.BigIntSpace = undefined; + const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu); + var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; + index_bigint.sub(low_bigint, min_bigint); + const start = index_bigint.toConst().toInt(u10) catch unreachable; + index_bigint.sub(high_bigint, min_bigint); + const end = @as(u11, index_bigint.toConst().toInt(u10) catch unreachable) + 1; + @memset(table[start..end], @intCast(cg.mir_instructions.len)); + } + } + + for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand, .{}); + + try cg.genBodyBlock(case.body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + } + if (switch_br.else_body_len > 0) { + const else_body = cases_it.elseBody(); + + const else_deaths = liveness.deaths.len - 1; + for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand, .{}); + + cg.performReloc(else_reloc); + if (is_loop) { + const loop_switch_data = cg.loop_switches.getPtr(inst).?; + for (loop_switch_data.else_relocs.forward.items) |reloc| cg.performReloc(reloc); + loop_switch_data.else_relocs.forward.deinit(cg.gpa); + loop_switch_data.else_relocs = .{ .backward = @intCast(cg.mir_instructions.len) }; + } + for (cg.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) { + entry.* = @intCast(cg.mir_instructions.len); + }; + + try cg.genBodyBlock(else_body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + } + return; + } + + var cases_it = switch_br.iterateCases(); + while (cases_it.next()) |case| { + const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); + defer allocator.free(relocs); + + try cg.spillEflagsIfOccupied(); + for (case.items, relocs[0..case.items.len]) |item, *reloc| { + const item_mcv = try cg.resolveInst(item); + const cc: Condition = switch (condition) { + .eflags => |cc| switch (item_mcv.immediate) { + 0 => cc.negate(), + 1 => cc, + else => unreachable, + }, + else => cc: { + var cond_temp = try cg.tempInit(condition_ty, condition); + var item_temp = try cg.tempInit(condition_ty, item_mcv); + const cc_temp = cond_temp.cmpInts(.eq, &item_temp, cg) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try item_temp.die(cg); + const cc = cc_temp.tracking(cg).short.eflags; + try cc_temp.die(cg); + try cond_temp.die(cg); + try cg.resetTemps(@enumFromInt(0)); + break :cc cc; + }, + }; + reloc.* = try cg.asmJccReloc(cc, undefined); + } + + for (case.ranges, relocs[case.items.len..]) |range, *reloc| { + var cond_temp = try cg.tempInit(condition_ty, condition); + const min_mcv = try cg.resolveInst(range[0]); + const max_mcv = try cg.resolveInst(range[1]); + // `null` means always false. + const lt_min = cc: switch (condition) { + .eflags => |cc| switch (min_mcv.immediate) { + 0 => null, // condition never <0 + 1 => cc.negate(), + else => unreachable, + }, + else => { + var min_temp = try cg.tempInit(condition_ty, min_mcv); + const cc_temp = cond_temp.cmpInts(.lt, &min_temp, cg) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try min_temp.die(cg); + const cc = cc_temp.tracking(cg).short.eflags; + try cc_temp.die(cg); + break :cc cc; + }, + }; + const lt_min_reloc = if (lt_min) |cc| r: { + break :r try cg.asmJccReloc(cc, undefined); + } else null; + // `null` means always true. + const lte_max = switch (condition) { + .eflags => |cc| switch (max_mcv.immediate) { + 0 => cc.negate(), + 1 => null, // condition always >=1 + else => unreachable, + }, + else => cc: { + var max_temp = try cg.tempInit(condition_ty, max_mcv); + const cc_temp = cond_temp.cmpInts(.lte, &max_temp, cg) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try max_temp.die(cg); + const cc = cc_temp.tracking(cg).short.eflags; + try cc_temp.die(cg); + break :cc cc; + }, + }; + try cond_temp.die(cg); + try cg.resetTemps(@enumFromInt(0)); + // "Success" case is in `reloc`.... + if (lte_max) |cc| { + reloc.* = try cg.asmJccReloc(cc, undefined); + } else { + reloc.* = try cg.asmJmpReloc(undefined); + } + // ...and "fail" case falls through to next checks. + if (lt_min_reloc) |r| cg.performReloc(r); + } + + cg.checkInvariantsAfterAirInst(); + + // The jump to skip this case if the conditions all failed. + const skip_case_reloc = try cg.asmJmpReloc(undefined); + + for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand, .{}); + + // Relocate all success cases to the body we're about to generate. + for (relocs) |reloc| cg.performReloc(reloc); + try cg.genBodyBlock(case.body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + + // Relocate the "skip" branch to fall through to the next case. + cg.performReloc(skip_case_reloc); + } + if (switch_br.else_body_len > 0) { + const else_body = cases_it.elseBody(); + + const else_deaths = liveness.deaths.len - 1; + for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand, .{}); + + try cg.genBodyBlock(else_body); + try cg.restoreState(state, &.{}, .{ + .emit_instructions = false, + .update_tracking = true, + .resurrect = true, + .close_scope = true, + }); + } +} + +fn airSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void { + const switch_br = self.air.unwrapSwitch(inst); + const condition = try self.resolveInst(switch_br.operand); + + // If the condition dies here in this switch instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + const condition_dies = self.liveness.operandDies(inst, 0); + if (condition_dies) { + if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); + } + try self.lowerSwitchBr(inst, switch_br, condition, condition_dies, false); + + // We already took care of pl_op.operand earlier, so there's nothing left to do +} + +fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void { + const switch_br = self.air.unwrapSwitch(inst); + const condition = try self.resolveInst(switch_br.operand); + + const mat_cond = if (condition.isModifiable() and + self.reuseOperand(inst, switch_br.operand, 0, condition)) + condition + else mat_cond: { + const mat_cond = try self.allocRegOrMem(inst, true); + try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{}); + break :mat_cond mat_cond; + }; + self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(mat_cond)); + + // If the condition dies here in this switch instruction, process + // that death now instead of later as this has an effect on + // whether it needs to be spilled in the branches + if (self.liveness.operandDies(inst, 0)) { + if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); + } + + // Ensure a register is available for dispatch. + if (!mat_cond.isRegister()) _ = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + + const state = try self.saveState(); + + try self.loops.putNoClobber(self.gpa, inst, .{ + .state = state, + .target = @intCast(self.mir_instructions.len), + }); + defer assert(self.loops.remove(inst)); + + // Stop tracking block result without forgetting tracking info + try self.freeValue(mat_cond, .{}); + + try self.lowerSwitchBr(inst, switch_br, mat_cond, true, true); + + try self.processDeath(inst, .{}); +} + +fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void { + const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; + + const block_ty = self.typeOfIndex(br.block_inst); + const loop_data = self.loops.getPtr(br.block_inst).?; + const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; + { + try self.getValue(block_tracking.short, null); + const src_mcv = try self.resolveInst(br.operand); + + if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { + try self.getValue(block_tracking.short, br.block_inst); + // .long = .none to avoid merging operand and block result stack frames. + const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; + try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*); + for (current_tracking.getRegs()) |src_reg| self.register_manager.freeReg(src_reg); + } else { + try self.getValue(block_tracking.short, br.block_inst); + try self.genCopy(block_ty, block_tracking.short, try self.resolveInst(br.operand), .{}); + } + } + + // Process operand death so that it is properly accounted for in the State below. + if (self.liveness.operandDies(inst, 0)) { + if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); + } + + try self.restoreState(loop_data.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + + if (self.loop_switches.getPtr(br.block_inst)) |table| { + const condition_ty = self.typeOf(br.operand); + const unsigned_condition_ty = try self.pt.intType(.unsigned, self.intInfo(condition_ty).?.bits); + const condition_mcv = block_tracking.short; + try self.spillEflagsIfOccupied(); + if (table.min.orderAgainstZero(self.pt.zcu).compare(.neq)) try self.genBinOpMir( + .{ ._, .sub }, + condition_ty, + condition_mcv, + .{ .air_ref = Air.internedToRef(table.min.toIntern()) }, + ); + switch (table.else_relocs) { + .@"unreachable" => {}, + .forward => |*else_relocs| { + var cond_temp = try self.tempInit(unsigned_condition_ty, condition_mcv); + var table_max_temp = try self.tempFromValue(try self.pt.intValue(unsigned_condition_ty, table.len - 1)); + const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try cond_temp.die(self); + try table_max_temp.die(self); + try else_relocs.append(self.gpa, try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined)); + try cc_temp.die(self); + }, + .backward => |else_reloc| { + var cond_temp = try self.tempInit(unsigned_condition_ty, condition_mcv); + var table_max_temp = try self.tempFromValue(try self.pt.intValue(unsigned_condition_ty, table.len - 1)); + const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) { + error.SelectFailed => unreachable, + else => |e| return e, + }; + try cond_temp.die(self); + try table_max_temp.die(self); + _ = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, else_reloc); + try cc_temp.die(self); + }, + } + { + const condition_index_reg = if (condition_mcv.isRegister()) condition_mcv.getReg().? else cond: { + const condition_index_reg = + RegisterManager.regAtTrackedIndex(@intCast(loop_data.state.free_registers.findFirstSet().?)); + try self.genSetReg(condition_index_reg, condition_ty, condition_mcv, .{}); + break :cond condition_index_reg; + }; + const condition_index_lock = self.register_manager.lockReg(condition_index_reg); + defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock); + try self.truncateRegister(condition_ty, condition_index_reg); + const ptr_size = @divExact(self.target.ptrBitWidth(), 8); + try self.asmMemory(.{ ._mp, .j }, .{ + .base = .table, + .mod = .{ .rm = .{ + .size = .ptr, + .index = condition_index_reg.toSize(.ptr, self.target), + .scale = .fromFactor(@intCast(ptr_size)), + .disp = @intCast(table.start * ptr_size), + } }, + }); + } + + return self.finishAir(inst, .none, .{ br.operand, .none, .none }); + } + + // Emit a jump with a relocation. It will be patched up after the block ends. + // Leave the jump offset undefined + _ = try self.asmJmpReloc(loop_data.target); + + // Stop tracking block result without forgetting tracking info + try self.freeValue(block_tracking.short, .{ .emit_instructions = false }); +} + +fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void { + const next_inst: u32 = @intCast(self.mir_instructions.len); + switch (self.mir_instructions.items(.tag)[reloc]) { + .j => {}, + .pseudo => switch (self.mir_instructions.items(.ops)[reloc]) { + .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {}, + else => unreachable, + }, + .lea => switch (self.mir_instructions.items(.ops)[reloc]) { + .rm => { + const rx = self.mir_instructions.items(.data)[reloc].rx; + assert(rx.fixes == ._); + const mem_info: Mir.Memory.Info = @bitCast( + self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "info").?], + ); + assert(mem_info.base == .rip_inst); + self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "base").?] = next_inst; + return; + }, + else => unreachable, + }, + else => unreachable, + } + self.mir_instructions.items(.data)[reloc].inst.inst = next_inst; +} + +fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; + + const block_ty = self.typeOfIndex(br.block_inst); + const block_unused = + !block_ty.hasRuntimeBitsIgnoreComptime(zcu) or self.liveness.isUnused(br.block_inst); + const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; + const block_data = self.blocks.getPtr(br.block_inst).?; + const first_br = block_data.relocs.items.len == 0; + const block_result = result: { + if (block_unused) break :result .none; + + if (!first_br) try self.getValue(block_tracking.short, null); + const src_mcv = try self.resolveInst(br.operand); + + if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { + if (first_br) break :result src_mcv; + + try self.getValue(block_tracking.short, br.block_inst); + try InstTracking.materializeUnsafe( + // .long = .none to avoid merging operand and block result stack frames. + .{ .long = .none, .short = src_mcv }, + self, + br.block_inst, + block_tracking.*, + ); + try self.freeValue(src_mcv, .{}); + break :result block_tracking.short; + } + + const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: { + try self.getValue(block_tracking.short, br.block_inst); + break :dst block_tracking.short; + }; + try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand), .{}); + break :result dst_mcv; + }; + + // Process operand death so that it is properly accounted for in the State below. + if (self.liveness.operandDies(inst, 0)) { + if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst, .{}); + } + + if (first_br) { + block_tracking.* = .init(block_result); + try self.saveRetroactiveState(&block_data.state); + } else try self.restoreState(block_data.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + + // Emit a jump with a relocation. It will be patched up after the block ends. + // Leave the jump offset undefined + const jmp_reloc = try self.asmJmpReloc(undefined); + try block_data.relocs.append(self.gpa, jmp_reloc); + + // Stop tracking block result without forgetting tracking info + try self.freeValue(block_tracking.short, .{ .emit_instructions = false }); +} + +fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { + @setEvalBranchQuota(1_100); + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Asm, ty_pl.payload); + const outputs_len = extra.data.flags.outputs_len; + var extra_i: usize = extra.end; + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..outputs_len]); + extra_i += outputs.len; + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); + extra_i += inputs.len; + + var result: MCValue = .none; + var args: std.array_list.Managed(MCValue) = .init(self.gpa); + try args.ensureTotalCapacity(outputs.len + inputs.len); + defer { + for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{ + .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue, + }); + args.deinit(); + } + var arg_map: std.StringHashMap(u8) = .init(self.gpa); + try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len)); + defer arg_map.deinit(); + + var outputs_extra_i = extra_i; + for (outputs) |output| { + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + const maybe_inst = switch (output) { + .none => inst, + else => null, + }; + const ty = switch (output) { + .none => self.typeOfIndex(inst), + else => self.typeOf(output).childType(zcu), + }; + const is_read = switch (constraint[0]) { + '=' => false, + '+' => read: { + if (output == .none) return self.fail( + "read-write constraint unsupported for asm result: '{s}'", + .{constraint}, + ); + break :read true; + }, + else => return self.fail("invalid constraint: '{s}'", .{constraint}), + }; + const is_early_clobber = constraint[1] == '&'; + const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..]; + const arg_mcv: MCValue = arg_mcv: { + const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or + std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x")) + registerAlias( + self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) { + 'r' => abi.RegisterClass.gp, + 'f' => abi.RegisterClass.x87, + 'x' => abi.RegisterClass.sse, + else => unreachable, + }) orelse return self.fail("ran out of registers lowering inline asm", .{}), + @intCast(ty.abiSize(zcu)), + ) + else if (std.mem.eql(u8, rest, "m")) + if (output != .none) null else return self.fail( + "memory constraint unsupported for asm result: '{s}'", + .{constraint}, + ) + else if (std.mem.eql(u8, rest, "g") or + std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or + std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r")) + self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse + if (output != .none) + null + else + return self.fail("ran out of registers lowering inline asm", .{}) + else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}")) + parseRegName(rest["{".len .. rest.len - "}".len]) orelse + return self.fail("invalid register constraint: '{s}'", .{constraint}) + else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { + const index = std.fmt.charToDigit(rest[0], 10) catch unreachable; + if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{ + constraint, + }); + break :arg_mcv args.items[index]; + } else return self.fail("invalid constraint: '{s}'", .{constraint}); + break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { + const ptr_mcv = try self.resolveInst(output); + switch (ptr_mcv) { + .immediate => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| + break :arg ptr_mcv.deref(), + .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(), + else => {}, + } + break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(.usize, ptr_mcv) } }; + }; + }; + if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |tracked_index| { + try self.register_manager.getRegIndex(tracked_index, if (output == .none) inst else null); + _ = self.register_manager.lockRegIndexAssumeUnused(tracked_index); + }; + if (!std.mem.eql(u8, name, "_")) + arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); + args.appendAssumeCapacity(arg_mcv); + if (output == .none) result = arg_mcv; + if (is_read) try self.load(arg_mcv, self.typeOf(output), .{ .air_ref = output }); + } + + for (inputs) |input| { + const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(input_bytes, 0); + const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + const ty = self.typeOf(input); + const input_mcv = try self.resolveInst(input); + const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or + std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x")) + arg: { + const rc = switch (constraint[0]) { + 'r' => abi.RegisterClass.gp, + 'f' => abi.RegisterClass.x87, + 'x' => abi.RegisterClass.sse, + else => unreachable, + }; + if (input_mcv.isRegister() and + rc.isSet(RegisterManager.indexOfRegIntoTracked(input_mcv.getReg().?).?)) + break :arg input_mcv; + const reg = try self.register_manager.allocReg(null, rc); + try self.genSetReg(reg, ty, input_mcv, .{}); + break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) }; + } else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n")) + switch (input_mcv) { + .immediate => |imm| .{ .immediate = imm }, + else => return self.fail("immediate operand requires comptime value: '{s}'", .{ + constraint, + }), + } + else if (std.mem.eql(u8, constraint, "m")) arg: { + switch (input_mcv) { + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| + break :arg input_mcv, + .indirect, .load_frame => break :arg input_mcv, + .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, + else => { + const temp_mcv = try self.allocTempRegOrMem(ty, false); + try self.genCopy(ty, temp_mcv, input_mcv, .{}); + break :arg temp_mcv; + }, + } + const addr_reg = self.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse { + const temp_mcv = try self.allocTempRegOrMem(ty, false); + try self.genCopy(ty, temp_mcv, input_mcv, .{}); + break :arg temp_mcv; + }; + try self.genSetReg(addr_reg, .usize, input_mcv.address(), .{}); + break :arg .{ .indirect = .{ .reg = addr_reg } }; + } else if (std.mem.eql(u8, constraint, "g") or + std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or + std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r")) + arg: { + switch (input_mcv) { + .register, .indirect, .load_frame => break :arg input_mcv, + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| + break :arg input_mcv, + else => {}, + } + const temp_mcv = try self.allocTempRegOrMem(ty, true); + try self.genCopy(ty, temp_mcv, input_mcv, .{}); + break :arg temp_mcv; + } else if (std.mem.eql(u8, constraint, "X")) + input_mcv + else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: { + const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse + return self.fail("invalid register constraint: '{s}'", .{constraint}); + try self.register_manager.getReg(reg, null); + try self.genSetReg(reg, ty, input_mcv, .{}); + break :arg .{ .register = reg }; + } else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: { + const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable; + if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint}); + try self.genCopy(ty, args.items[index], input_mcv, .{}); + break :arg args.items[index]; + } else return self.fail("invalid constraint: '{s}'", .{constraint}); + if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { + _ = self.register_manager.lockReg(reg); + }; + if (!std.mem.eql(u8, name, "_")) + arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); + args.appendAssumeCapacity(arg_mcv); + } + + const ip = &zcu.intern_pool; + const aggregate = ip.indexToKey(extra.data.clobbers).aggregate; + const struct_type: Type = .fromInterned(aggregate.ty); + switch (aggregate.storage) { + .elems => |elems| for (elems, 0..) |elem, i| switch (elem) { + .bool_true => { + const clobber = struct_type.structFieldName(i, zcu).toSlice(ip).?; + assert(clobber.len != 0); + + if (std.mem.eql(u8, clobber, "memory") or + std.mem.eql(u8, clobber, "fpsr") or + std.mem.eql(u8, clobber, "fpcr") or + std.mem.eql(u8, clobber, "mxcsr") or + std.mem.eql(u8, clobber, "dirflag")) + { + // ok, sure + } else if (std.mem.eql(u8, clobber, "cc") or + std.mem.eql(u8, clobber, "flags") or + std.mem.eql(u8, clobber, "eflags") or + std.mem.eql(u8, clobber, "rflags")) + { + try self.spillEflagsIfOccupied(); + } else { + try self.register_manager.getReg(parseRegName(clobber) orelse + return self.fail("invalid clobber: '{s}'", .{clobber}), null); + } + }, + .bool_false => continue, + else => unreachable, + }, + .repeated_elem => |elem| switch (elem) { + .bool_true => @panic("TODO"), + .bool_false => {}, + else => unreachable, + }, + .bytes => @panic("TODO"), + } + + const Label = struct { + target: Mir.Inst.Index = undefined, + pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, + + const Kind = enum { definition, reference }; + + fn isValid(kind: Kind, name: []const u8) bool { + for (name, 0..) |c, i| switch (c) { + else => return false, + '$' => if (i == 0) return false, + '.' => {}, + '0'...'9' => if (i == 0) switch (kind) { + .definition => if (name.len != 1) return false, + .reference => { + if (name.len != 2) return false; + switch (name[1]) { + else => return false, + 'B', 'F', 'b', 'f' => {}, + } + }, + }, + '@', 'A'...'Z', '_', 'a'...'z' => {}, + }; + return name.len > 0; + } + }; + var labels: std.StringHashMapUnmanaged(Label) = .empty; + defer { + var label_it = labels.valueIterator(); + while (label_it.next()) |label| label.pending_relocs.deinit(self.gpa); + labels.deinit(self.gpa); + } + + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; + var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;"); + next_line: while (line_it.next()) |line| { + var mnem_it = std.mem.tokenizeAny(u8, line, " \t"); + var prefix: encoder.Instruction.Prefix = .none; + const mnem_str = while (mnem_it.next()) |mnem_str| { + if (mnem_str[0] == '#') continue :next_line; + if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line; + if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) |pre| { + if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str}); + prefix = pre; + continue; + } + if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str; + const label_name = mnem_str[0 .. mnem_str.len - ":".len]; + if (!Label.isValid(.definition, label_name)) + return self.fail("invalid label: '{s}'", .{label_name}); + const label_gop = try labels.getOrPut(self.gpa, label_name); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else { + const anon = std.ascii.isDigit(label_name[0]); + if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0) + return self.fail("redefined label: '{s}'", .{label_name}); + for (label_gop.value_ptr.pending_relocs.items) |pending_reloc| + self.performReloc(pending_reloc); + if (anon) + label_gop.value_ptr.pending_relocs.clearRetainingCapacity() + else + label_gop.value_ptr.pending_relocs.clearAndFree(self.gpa); + } + label_gop.value_ptr.target = @intCast(self.mir_instructions.len); + } else continue; + if (mnem_str[0] == '.') { + if (prefix != .none) return self.fail("prefixed directive: '{s} {s}'", .{ @tagName(prefix), mnem_str }); + prefix = .directive; + } + + var mnem_size: struct { + op_has_size: std.StaticBitSet(4), + size: Memory.Size, + used: bool, + fn init(size: ?Memory.Size) @This() { + return .{ + .op_has_size = if (size) |_| .initFull() else .initEmpty(), + .size = size orelse .none, + .used = false, + }; + } + fn use(size: *@This(), op_index: usize) ?Memory.Size { + if (!size.op_has_size.isSet(op_index)) return null; + size.used = true; + return size.size; + } + } = .init(if (prefix == .directive) + null + else if (std.mem.endsWith(u8, mnem_str, "b")) + .byte + else if (std.mem.endsWith(u8, mnem_str, "w")) + .word + else if (std.mem.endsWith(u8, mnem_str, "l")) + .dword + else if (std.mem.endsWith(u8, mnem_str, "q") and + (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq"))) + .qword + else if (std.mem.endsWith(u8, mnem_str, "t")) + .tbyte + else + null); + var mnem_tag = while (true) break std.meta.stringToEnum( + encoder.Instruction.Mnemonic, + mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size.size != .none)], + ) orelse if (mnem_size.size != .none) { + mnem_size = .init(null); + continue; + } else return self.fail("invalid mnemonic: '{s}'", .{mnem_str}); + fixed_mnem_size: { + const fixed_mnem_size: Memory.Size = switch (mnem_tag) { + .clflush => .byte, + .fldcw, .fnstcw, .fstcw, .fnstsw, .fstsw => .word, + .fldenv, .fnstenv, .fstenv => .none, + .frstor, .fsave, .fnsave, .fxrstor, .fxrstor64, .fxsave, .fxsave64 => .none, + .in => { + mnem_size.op_has_size.unset(0); + break :fixed_mnem_size; + }, + .invlpg => .none, + .invpcid => .xword, + .ldmxcsr, .stmxcsr, .vldmxcsr, .vstmxcsr => .dword, + .out => { + mnem_size.op_has_size.unset(1); + break :fixed_mnem_size; + }, + else => break :fixed_mnem_size, + }; + if (mnem_size.size != .none and mnem_size.size != fixed_mnem_size) + return self.fail("invalid size: '{s}'", .{mnem_str}); + mnem_size = .init(fixed_mnem_size); + } + + var ops: [4]Operand = @splat(.none); + var ops_len: usize = 0; + + var last_op = false; + var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ','); + next_op: for (&ops, 0..) |*op, op_index| { + const op_str = while (!last_op) { + const full_str = op_it.next() orelse break :next_op; + const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse + std.mem.indexOf(u8, full_str, "//")) |comment| + code: { + last_op = true; + break :code full_str[0..comment]; + } else full_str; + const trim_str = std.mem.trim(u8, code_str, " \t*"); + if (trim_str.len > 0) break trim_str; + } else break; + if (std.mem.startsWith(u8, op_str, "%%")) { + const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':'); + const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse + return self.fail("invalid register: '{s}'", .{op_str}); + if (colon) |colon_pos| { + const disp = std.fmt.parseInt(i32, op_str[colon_pos + ":".len ..], 0) catch + return self.fail("invalid displacement: '{s}'", .{op_str}); + op.* = .{ .mem = .{ + .base = .{ .reg = reg }, + .mod = .{ .rm = .{ + .size = mnem_size.use(op_index) orelse + return self.fail("unknown size: '{s}'", .{op_str}), + .disp = disp, + } }, + } }; + } else { + if (mnem_size.use(op_index)) |size| if (reg.size().bitSize(self.target) != size.bitSize(self.target)) + return self.fail("invalid register size: '{s}'", .{op_str}); + op.* = .{ .reg = reg }; + } + } else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) { + const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':'); + const modifier = if (colon) |colon_pos| + op_str[colon_pos + ":".len .. op_str.len - "]".len] + else + ""; + op.* = switch (args.items[ + arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse + return self.fail("no matching constraint: '{s}'", .{op_str}) + ]) { + .immediate => |imm| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c")) + .{ .imm = .u(imm) } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .register => |reg| if (std.mem.eql(u8, modifier, "")) + .{ .reg = if (mnem_size.use(op_index)) |size| reg.toSize(size, self.target) else reg } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .memory => |addr| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P")) + .{ .mem = .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = mnem_size.use(op_index) orelse + return self.fail("unknown size: '{s}'", .{op_str}), + .disp = @intCast(@as(i64, @bitCast(addr))), + } }, + } } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .indirect => |reg_off| if (std.mem.eql(u8, modifier, "")) + .{ .mem = .{ + .base = .{ .reg = reg_off.reg }, + .mod = .{ .rm = .{ + .size = mnem_size.use(op_index) orelse + return self.fail("unknown size: '{s}'", .{op_str}), + .disp = reg_off.off, + } }, + } } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .load_frame => |frame_addr| if (std.mem.eql(u8, modifier, "")) + .{ .mem = .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = mnem_size.use(op_index) orelse + return self.fail("unknown size: '{s}'", .{op_str}), + .disp = frame_addr.off, + } }, + } } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .lea_nav => |nav| if (std.mem.eql(u8, modifier, "P")) + .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_nav = nav }) } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .lea_uav => |uav| if (std.mem.eql(u8, modifier, "P")) + .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_uav = uav }) } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .lea_lazy_sym => |lazy_sym| if (std.mem.eql(u8, modifier, "P")) + .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_lazy_sym = lazy_sym }) } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + .lea_extern_func => |extern_func| if (std.mem.eql(u8, modifier, "P")) + .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_extern_func = extern_func }) } + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + else => return self.fail("invalid constraint: '{s}'", .{op_str}), + }; + } else if (std.mem.startsWith(u8, op_str, "$")) { + op.* = if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| + .{ .imm = .u(u) } + else |_| if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| + .{ .imm = .s(s) } + else |_| + return self.fail("invalid immediate: '{s}'", .{op_str}); + } else if (std.mem.endsWith(u8, op_str, ")")) { + const open = std.mem.indexOfScalar(u8, op_str, '(') orelse + return self.fail("invalid operand: '{s}'", .{op_str}); + var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ','); + const base_str = sib_it.next() orelse + return self.fail("invalid memory operand: '{s}'", .{op_str}); + if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%")) + return self.fail("invalid memory operand: '{s}'", .{op_str}); + const index_str = sib_it.next() orelse ""; + if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%")) + return self.fail("invalid memory operand: '{s}'", .{op_str}); + const scale_str = sib_it.next() orelse ""; + if (index_str.len == 0 and scale_str.len > 0) + return self.fail("invalid memory operand: '{s}'", .{op_str}); + const scale: Memory.Scale = if (scale_str.len > 0) + switch (std.fmt.parseInt(u4, scale_str, 10) catch + return self.fail("invalid scale: '{s}'", .{op_str})) { + 1 => .@"1", + 2 => .@"2", + 4 => .@"4", + 8 => .@"8", + else => return self.fail("invalid scale: '{s}'", .{op_str}), + } + else + .@"1"; + if (sib_it.next()) |_| return self.fail("invalid memory operand: '{s}'", .{op_str}); + op.* = if (std.mem.eql(u8, base_str, "%%dx") and index_str.len == 0) + .{ .reg = .dx } + else if (std.mem.eql(u8, base_str, "%%rip") and index_str.len == 0 and + Label.isValid(.reference, op_str[0..open])) + op: { + const anon = std.ascii.isDigit(op_str[0]); + const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else open]); + if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) + return self.fail("undefined label: '{s}'", .{op_str}); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; + const pending_relocs = &label_gop.value_ptr.pending_relocs; + if (if (anon) + op_str[1] == 'f' or op_str[1] == 'F' + else + !label_gop.found_existing or pending_relocs.items.len > 0) + try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len)); + break :op .{ .mem = .{ .base = .{ .rip_inst = label_gop.value_ptr.target } } }; + } else .{ .mem = .{ + .base = if (base_str.len > 0) + .{ .reg = parseRegName(base_str["%%".len..]) orelse + return self.fail("invalid base register: '{s}'", .{base_str}) } + else + .none, + .mod = .{ .rm = .{ + .size = mnem_size.use(op_index) orelse return self.fail("unknown size: '{s}'", .{op_str}), + .index = if (index_str.len > 0) + parseRegName(index_str["%%".len..]) orelse + return self.fail("invalid index register: '{s}'", .{op_str}) + else + .none, + .scale = scale, + .disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and + std.mem.endsWith(u8, op_str[0..open], "]")) + disp: { + const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':'); + const modifier = if (colon) |colon_pos| + op_str[colon_pos + ":".len .. open - "]".len] + else + ""; + break :disp switch (args.items[ + arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse + return self.fail("no matching constraint: '{s}'", .{op_str}) + ]) { + .immediate => |imm| if (std.mem.eql(u8, modifier, "") or + std.mem.eql(u8, modifier, "c")) + std.math.cast(i32, @as(i64, @bitCast(imm))) orelse + return self.fail("invalid displacement: '{s}'", .{op_str}) + else + return self.fail("invalid modifier: '{s}'", .{modifier}), + else => return self.fail("invalid constraint: '{s}'", .{op_str}), + }; + } else if (open > 0) + std.fmt.parseInt(i32, op_str[0..open], 0) catch + return self.fail("invalid displacement: '{s}'", .{op_str}) + else + 0, + } }, + } }; + } else if (Label.isValid(.reference, op_str)) { + const anon = std.ascii.isDigit(op_str[0]); + const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]); + if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) + return self.fail("undefined label: '{s}'", .{op_str}); + if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; + const pending_relocs = &label_gop.value_ptr.pending_relocs; + if (if (anon) + op_str[1] == 'f' or op_str[1] == 'F' + else + !label_gop.found_existing or pending_relocs.items.len > 0) + try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len)); + op.* = .{ .inst = label_gop.value_ptr.target }; + } else return self.fail("invalid operand: '{s}'", .{op_str}); + ops_len += 1; + } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str}); + + // convert from att syntax to intel syntax + std.mem.reverse(Operand, ops[0..ops_len]); + if (mnem_size.size != .none and !mnem_size.used) { + comptime var max_mnem_len: usize = 0; + inline for (@typeInfo(encoder.Instruction.Mnemonic).@"enum".fields) |mnem| + max_mnem_len = @max(mnem.name.len, max_mnem_len); + var intel_mnem_buf: [max_mnem_len + 1]u8 = undefined; + const intel_mnem_str = std.fmt.bufPrint(&intel_mnem_buf, "{s}{c}", .{ + @tagName(mnem_tag), + @as(u8, switch (mnem_size.size) { + .byte => 'b', + .word => 'w', + .dword => 'd', + .qword => 'q', + .tbyte => 't', + else => unreachable, + }), + }) catch unreachable; + if (std.meta.stringToEnum(encoder.Instruction.Mnemonic, intel_mnem_str)) |intel_mnem_tag| mnem_tag = intel_mnem_tag; + } + const mnem_name = @tagName(mnem_tag); + const mnem_fixed_tag: Mir.Inst.FixedTag = if (prefix == .directive) + .{ ._, .pseudo } + else for (std.enums.values(Mir.Inst.Fixes)) |fixes| { + const fixes_name = @tagName(fixes); + const space_index = std.mem.indexOfScalar(u8, fixes_name, ' '); + const fixes_prefix = if (space_index) |index| + std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..index]).? + else + .none; + if (fixes_prefix != prefix) continue; + const pattern = fixes_name[if (space_index) |index| index + " ".len else 0..]; + const wildcard_index = std.mem.indexOfScalar(u8, pattern, '_').?; + const mnem_prefix = pattern[0..wildcard_index]; + const mnem_suffix = pattern[wildcard_index + "_".len ..]; + if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue; + if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue; + break .{ fixes, std.meta.stringToEnum( + Mir.Inst.Tag, + mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len], + ) orelse continue }; + } else { + assert(prefix != .none); // no combination of fixes produced a known mnemonic + return self.fail("invalid prefix for mnemonic: '{s} {s}'", .{ + @tagName(prefix), mnem_name, + }); + }; + + (if (prefix == .directive) switch (mnem_tag) { + .@".cfi_def_cfa" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) + self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, ops[0].reg, ops[1].imm) + else + error.InvalidInstruction, + .@".cfi_def_cfa_register" => if (ops[0] == .reg and ops[1] == .none) + self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, ops[0].reg) + else + error.InvalidInstruction, + .@".cfi_def_cfa_offset" => if (ops[0] == .imm and ops[1] == .none) + self.asmPseudoImmediate(.pseudo_cfi_def_cfa_offset_i_s, ops[0].imm) + else + error.InvalidInstruction, + .@".cfi_adjust_cfa_offset" => if (ops[0] == .imm and ops[1] == .none) + self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, ops[0].imm) + else + error.InvalidInstruction, + .@".cfi_offset" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) + self.asmPseudoRegisterImmediate(.pseudo_cfi_offset_ri_s, ops[0].reg, ops[1].imm) + else + error.InvalidInstruction, + .@".cfi_val_offset" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) + self.asmPseudoRegisterImmediate(.pseudo_cfi_val_offset_ri_s, ops[0].reg, ops[1].imm) + else + error.InvalidInstruction, + .@".cfi_rel_offset" => if (ops[0] == .reg and ops[1] == .imm and ops[2] == .none) + self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, ops[0].reg, ops[1].imm) + else + error.InvalidInstruction, + .@".cfi_register" => if (ops[0] == .reg and ops[1] == .reg and ops[2] == .none) + self.asmPseudoRegisterRegister(.pseudo_cfi_register_rr, ops[0].reg, ops[1].reg) + else + error.InvalidInstruction, + .@".cfi_restore" => if (ops[0] == .reg and ops[1] == .none) + self.asmPseudoRegister(.pseudo_cfi_restore_r, ops[0].reg) + else + error.InvalidInstruction, + .@".cfi_undefined" => if (ops[0] == .reg and ops[1] == .none) + self.asmPseudoRegister(.pseudo_cfi_undefined_r, ops[0].reg) + else + error.InvalidInstruction, + .@".cfi_same_value" => if (ops[0] == .reg and ops[1] == .none) + self.asmPseudoRegister(.pseudo_cfi_same_value_r, ops[0].reg) + else + error.InvalidInstruction, + .@".cfi_remember_state" => if (ops[0] == .none) + self.asmPseudo(.pseudo_cfi_remember_state_none) + else + error.InvalidInstruction, + .@".cfi_restore_state" => if (ops[0] == .none) + self.asmPseudo(.pseudo_cfi_restore_state_none) + else + error.InvalidInstruction, + .@".cfi_escape" => error.InvalidInstruction, + else => unreachable, + } else self.asmOps(mnem_fixed_tag, ops)) catch |err| switch (err) { + error.InvalidInstruction => return self.fail( + "invalid instruction: '{s} {s} {s} {s} {s}'", + .{ + mnem_str, + @tagName(ops[0]), + @tagName(ops[1]), + @tagName(ops[2]), + @tagName(ops[3]), + }, + ), + else => |e| return e, + }; + } + + var label_it = labels.iterator(); + while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0) + return self.fail("undefined label: '{s}'", .{label.key_ptr.*}); + + for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]); + const constraint = + std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]), 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + if (output == .none) continue; + if (arg_mcv != .register) continue; + if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue; + try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv, .{}); + } + + simple: { + var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); + var buf_index: usize = 0; + for (outputs) |output| { + if (output == .none) continue; + + if (buf_index >= buf.len) break :simple; + buf[buf_index] = output; + buf_index += 1; + } + if (buf_index + inputs.len > buf.len) break :simple; + @memcpy(buf[buf_index..][0..inputs.len], inputs); + return self.finishAir(inst, result, buf); + } + var bt = self.liveness.iterateBigTomb(inst); + for (outputs) |output| if (output != .none) try self.feed(&bt, output); + for (inputs) |input| try self.feed(&bt, input); + return self.finishAirResult(inst, result); +} + +const MoveStrategy = union(enum) { + move_through_gpr, + load_store_through_frame, + load_store: Mir.Inst.FixedTag, + load_store_x87, + insert_extract: InsertExtract, + insert_or_extract_through_gpr: InsertExtract, + vex_insert_extract: InsertExtract, + + const InsertExtract = struct { + insert: Mir.Inst.FixedTag, + extract: Mir.Inst.FixedTag, + }; + + pub fn read(strat: MoveStrategy, cg: *CodeGen, dst_reg: Register, src_mem: Memory) !void { + switch (strat) { + .move_through_gpr => { + const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); + try cg.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), src_mem); + try cg.asmRegisterRegister(.{ ._d, .mov }, dst_reg, tmp_reg.to32()); + }, + .load_store_through_frame => { + const tmp_frame_index = try cg.allocFrameIndex(.init(.{ + .size = 16, + .alignment = .@"16", + })); + const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); + try cg.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), src_mem); + try cg.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = tmp_frame_index }, + .mod = .{ .rm = .{ .size = .dword } }, + }, tmp_reg.to32()); + try cg.asmRegisterMemory(.{ ._ps, .mova }, dst_reg, .{ + .base = .{ .frame = tmp_frame_index }, + .mod = .{ .rm = .{ .size = .xword } }, + }); + }, + .load_store => |tag| { + if (tag[0] == ._ps and tag[1] == .movl) try cg.asmRegisterRegister(.{ ._ps, .xor }, dst_reg, dst_reg); + try cg.asmRegisterMemory(tag, switch (tag[1]) { + else => dst_reg, + .lea => if (dst_reg.size().bitSize(cg.target) >= 32) dst_reg else dst_reg.to32(), + }, src_mem); + }, + .load_store_x87 => if (dst_reg != .st0 and cg.register_manager.isKnownRegFree(.st7)) { + try cg.asmMemory(.{ .f_, .ld }, src_mem); + switch (dst_reg) { + .st1, .st2, .st3, .st4, .st5, .st6 => try cg.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1)), + .st7 => try cg.asmOpOnly(.{ .f_cstp, .in }), + else => unreachable, + } + } else { + try cg.asmRegister(.{ .f_p, .st }, dst_reg); + try cg.asmMemory(.{ .f_, .ld }, src_mem); + switch (dst_reg) { + .st0 => {}, + .st1, .st2, .st3, .st4, .st5, .st6, .st7 => try cg.asmRegister(.{ .f_, .xch }, dst_reg), + else => unreachable, + } + }, + .insert_extract, .insert_or_extract_through_gpr => |ie| { + try cg.asmRegisterRegister(.{ .p_, .xor }, dst_reg, dst_reg); + try cg.asmRegisterMemoryImmediate(ie.insert, dst_reg, src_mem, .u(0)); + }, + .vex_insert_extract => |ie| { + try cg.asmRegisterRegisterRegister(.{ .vp_, .xor }, dst_reg, dst_reg, dst_reg); + try cg.asmRegisterRegisterMemoryImmediate(ie.insert, dst_reg, dst_reg, src_mem, .u(0)); + }, + } + } + pub fn write(strat: MoveStrategy, cg: *CodeGen, dst_mem: Memory, src_reg: Register) !void { + switch (strat) { + .move_through_gpr => { + const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); + try cg.asmRegisterRegister(.{ ._d, .mov }, tmp_reg.to32(), src_reg.to128()); + try cg.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.toSize(dst_mem.mod.rm.size, cg.target)); + }, + .load_store_through_frame => { + const tmp_frame_index = try cg.allocFrameIndex(.init(.{ + .size = 16, + .alignment = .@"16", + })); + try cg.asmMemoryRegister(.{ ._ps, .mova }, .{ + .base = .{ .frame = tmp_frame_index }, + .mod = .{ .rm = .{ .size = .xword } }, + }, src_reg.to128()); + const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); + try cg.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), .{ + .base = .{ .frame = tmp_frame_index }, + .mod = .{ .rm = .{ .size = .word } }, + }); + try cg.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.toSize(dst_mem.mod.rm.size, cg.target)); + }, + .load_store => |tag| try cg.asmMemoryRegister(tag, dst_mem, src_reg), + .load_store_x87 => if (cg.register_manager.isKnownRegFree(.st7)) { + try cg.asmRegister(.{ .f_, .ld }, src_reg); + try cg.asmMemory(.{ .f_p, .st }, dst_mem); + } else { + switch (src_reg) { + .st0 => {}, + .st1, .st2, .st3, .st4, .st5, .st6, .st7 => try cg.asmRegister(.{ .f_, .xch }, src_reg), + else => unreachable, + } + try cg.asmMemory(.{ .f_p, .st }, dst_mem); + try cg.asmMemory(.{ .f_, .ld }, dst_mem); + switch (src_reg) { + .st0 => {}, + .st1, .st2, .st3, .st4, .st5, .st6, .st7 => try cg.asmRegister(.{ .f_, .xch }, src_reg), + else => unreachable, + } + }, + .insert_extract, .vex_insert_extract => |ie| try cg.asmMemoryRegisterImmediate(ie.extract, dst_mem, src_reg, .u(0)), + .insert_or_extract_through_gpr => |ie| { + const tmp_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); + try cg.asmRegisterRegisterImmediate(ie.extract, tmp_reg.to32(), src_reg.to128(), .u(0)); + try cg.asmMemoryRegister(.{ ._, .mov }, dst_mem, tmp_reg.to16()); + }, + } + } +}; +fn moveStrategy(cg: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy { + const pt = cg.pt; + const zcu = pt.zcu; + switch (class) { + .general_purpose, .gphi, .segment => return .{ .load_store = .{ ._, .mov } }, + .x87 => return .load_store_x87, + .mmx => {}, + .sse => switch (ty.zigTypeTag(zcu)) { + else => { + const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .other), .none); + assert(std.mem.indexOfNone(abi.Class, classes, &.{ + .integer, .sse, .sseup, .memory, .float, .float_combine, + }) == null); + const abi_size = ty.abiSize(zcu); + if (abi_size < 4 or std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) { + 1 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ + .insert = .{ .vp_b, .insr }, + .extract = .{ .vp_b, .extr }, + } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ + .insert = .{ .p_b, .insr }, + .extract = .{ .p_b, .extr }, + } } else if (cg.hasFeature(.sse2)) .move_through_gpr else .load_store_through_frame, + 2 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ + .insert = .{ .vp_w, .insr }, + .extract = .{ .vp_w, .extr }, + } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ + .insert = .{ .p_w, .insr }, + .extract = .{ .p_w, .extr }, + } } else if (cg.hasFeature(.sse2)) .{ .insert_or_extract_through_gpr = .{ + .insert = .{ .p_w, .insr }, + .extract = .{ .p_w, .extr }, + } } else .load_store_through_frame, + 3...4 => return .{ .load_store = .{ if (cg.hasFeature(.avx)) + .v_d + else if (cg.hasFeature(.sse2)) + ._d + else + ._ss, .mov } }, + 5...8 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_q, .mov } + else if (cg.hasFeature(.sse2)) + .{ ._q, .mov } + else + .{ ._ps, .movl } }, + 9...16 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 17...32 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + } else switch (abi_size) { + 4 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_ss, .mov } + else + .{ ._ss, .mov } }, + 5...8 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_sd, .mov } + else if (cg.hasFeature(.sse2)) + .{ ._sd, .mov } + else + .{ ._ps, .movl } }, + 9...16 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_pd, if (aligned) .mova else .movu } + else if (cg.hasFeature(.sse2)) + .{ ._pd, if (aligned) .mova else .movu } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 17...32 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ .v_pd, if (aligned) .mova else .movu } }, + else => {}, + } + }, + .float => switch (ty.floatBits(cg.target)) { + 16 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ + .insert = .{ .vp_w, .insr }, + .extract = .{ .vp_w, .extr }, + } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ + .insert = .{ .p_w, .insr }, + .extract = .{ .p_w, .extr }, + } } else if (cg.hasFeature(.sse2)) .{ .insert_or_extract_through_gpr = .{ + .insert = .{ .p_w, .insr }, + .extract = .{ .p_w, .extr }, + } } else .load_store_through_frame, + 32 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_ss, .mov } + else + .{ ._ss, .mov } }, + 64 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_sd, .mov } + else if (cg.hasFeature(.sse2)) + .{ ._sd, .mov } + else + .{ ._ps, .movl } }, + 128 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + else => {}, + }, + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + .bool => switch (ty.vectorLen(zcu)) { + 1...8 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ + .insert = .{ .vp_b, .insr }, + .extract = .{ .vp_b, .extr }, + } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ + .insert = .{ .p_b, .insr }, + .extract = .{ .p_b, .extr }, + } } else if (cg.hasFeature(.sse2)) .move_through_gpr else .load_store_through_frame, + 9...16 => return if (cg.hasFeature(.avx)) .{ .vex_insert_extract = .{ + .insert = .{ .vp_w, .insr }, + .extract = .{ .vp_w, .extr }, + } } else if (cg.hasFeature(.sse4_1)) .{ .insert_extract = .{ + .insert = .{ .p_w, .insr }, + .extract = .{ .p_w, .extr }, + } } else if (cg.hasFeature(.sse2)) .{ .insert_or_extract_through_gpr = .{ + .insert = .{ .p_w, .insr }, + .extract = .{ .p_w, .extr }, + } } else .load_store_through_frame, + 17...32 => return .{ .load_store = .{ if (cg.hasFeature(.avx)) + .v_d + else if (cg.hasFeature(.sse2)) + ._d + else + ._ss, .mov } }, + 33...64 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_q, .mov } + else if (cg.hasFeature(.sse2)) + .{ ._q, .mov } + else + .{ ._ps, .movl } }, + 65...128 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 129...256 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + .int => switch (ty.childType(zcu).intInfo(zcu).bits) { + 1...8 => switch (ty.vectorLen(zcu)) { + 1...16 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 17...32 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + 9...16 => switch (ty.vectorLen(zcu)) { + 1...8 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 9...16 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + 17...32 => switch (ty.vectorLen(zcu)) { + 1...4 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 5...8 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + 33...64 => switch (ty.vectorLen(zcu)) { + 1...2 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 3...4 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + 65...128 => switch (ty.vectorLen(zcu)) { + 1 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 2 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + 129...256 => switch (ty.vectorLen(zcu)) { + 1 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + else => {}, + }, + .pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu)) + switch (ty.vectorLen(zcu)) { + 1...2 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 3...4 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + } + else + unreachable, + .float => switch (ty.childType(zcu).floatBits(cg.target)) { + 16 => switch (ty.vectorLen(zcu)) { + 1...8 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 9...16 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + 32 => switch (ty.vectorLen(zcu)) { + 1...4 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_ps, if (aligned) .mova else .movu } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 5...8 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ .v_ps, if (aligned) .mova else .movu } }, + else => {}, + }, + 64 => switch (ty.vectorLen(zcu)) { + 1...2 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ .v_pd, if (aligned) .mova else .movu } + else if (cg.hasFeature(.sse2)) + .{ ._pd, if (aligned) .mova else .movu } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 3...4 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ .v_pd, if (aligned) .mova else .movu } }, + else => {}, + }, + 80, 128 => switch (ty.vectorLen(zcu)) { + 1 => return .{ .load_store = if (cg.hasFeature(.avx)) + .{ if (aligned) .v_dqa else .v_dqu, .mov } + else if (cg.hasFeature(.sse2)) + .{ if (aligned) ._dqa else ._dqu, .mov } + else + .{ ._ps, if (aligned) .mova else .movu } }, + 2 => if (cg.hasFeature(.avx)) + return .{ .load_store = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, + else => {}, + }, + else => {}, + }, + else => {}, + }, + }, + .ip, .cr, .dr => {}, + } + return cg.fail("TODO moveStrategy for {f}", .{ty.fmt(pt)}); +} + +const CopyOptions = struct { + safety: bool = false, +}; + +fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void { + const pt = self.pt; + + const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + switch (dst_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register_overflow, + .register_mask, + .indirect_load_frame, + .lea_frame, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // unmodifiable destination + .register => |dst_reg| try self.genSetReg(dst_reg, ty, src_mcv, opts), + .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) { + .none, + .unreach, + .dead, + .undef, + .register_overflow, + .elementwise_args, + .reserved_frame, + => unreachable, + .immediate, + .register, + .register_offset, + .lea_frame, + => src_mcv.offset(-dst_reg_off.off), + else => .{ .register_offset = .{ + .reg = try self.copyToTmpRegister(ty, src_mcv), + .off = -dst_reg_off.off, + } }, + }, opts), + inline .register_pair, .register_triple, .register_quadruple => |dst_regs, dst_tag| { + const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = src_info: switch (src_mcv) { + .undef, .memory, .indirect, .load_frame => null, + .register => |src_reg| switch (dst_regs[0].class()) { + .general_purpose => switch (src_reg.class()) { + else => unreachable, + .sse => if (ty.abiSize(pt.zcu) <= 16) { + if (self.hasFeature(.avx)) { + try self.asmRegisterRegister(.{ .v_q, .mov }, dst_regs[0].to64(), src_reg.to128()); + try self.asmRegisterRegisterImmediate(.{ .vp_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1)); + } else if (self.hasFeature(.sse4_1)) { + try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128()); + try self.asmRegisterRegisterImmediate(.{ .p_q, .extr }, dst_regs[1].to64(), src_reg.to128(), .u(1)); + } else if (self.hasFeature(.sse2)) { + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[0].to64(), src_reg.to128()); + try self.asmRegisterRegister(.{ ._ps, .movhl }, tmp_reg.to128(), src_reg.to128()); + try self.asmRegisterRegister(.{ ._q, .mov }, dst_regs[1].to64(), tmp_reg.to128()); + } else { + const frame_index = try self.allocFrameIndex(.init(.{ + .size = 16, + .alignment = .@"16", + })); + try self.asmMemoryRegister(.{ ._ps, .mova }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .xword } }, + }, src_reg.to128()); + for (dst_regs, 0..) |dst_reg, dst_index| try self.asmRegisterMemory(.{ ._, .mov }, dst_reg.to64(), .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .qword, .disp = @intCast(8 * dst_index) } }, + }); + } + return; + } else unreachable, + }, + else => unreachable, + }, + dst_tag => |src_regs| { + var remaining: std.StaticBitSet(dst_regs.len) = .initFull(); + var hazard_regs = src_regs; + while (!remaining.eql(.initEmpty())) { + var remaining_it = remaining.iterator(.{}); + next: while (remaining_it.next()) |index| { + const dst_reg = dst_regs[index]; + const dst_id = dst_reg.id(); + const src_reg = hazard_regs[index]; + const src_id = src_reg.id(); + if (dst_id == src_id) { + remaining.unset(index); + continue; + } + var mir_tag: Mir.Inst.FixedTag = .{ ._, .mov }; + var hazard_it = remaining.iterator(.{}); + while (hazard_it.next()) |hazard_index| { + if (dst_id != hazard_regs[hazard_index].id()) continue; + for (dst_regs) |clobber_reg| { + if (src_id == clobber_reg.id()) break; + } else continue :next; + hazard_regs[hazard_index] = src_reg; + mir_tag = .{ ._g, .xch }; + } + remaining.unset(index); + try self.asmRegisterRegister(mir_tag, dst_reg.to64(), src_reg.to64()); + } + } + return; + }, + .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => { + const src_addr_reg = + (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); + const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); + errdefer self.register_manager.unlockReg(src_addr_lock); + + try self.genSetReg(src_addr_reg, .usize, src_mcv.address(), opts); + break :src_info .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; + }, + .air_ref => |src_ref| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref), opts), + else => return self.fail("TODO implement genCopy for {s} of {f}", .{ + @tagName(src_mcv), ty.fmt(pt), + }), + }; + defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); + + for ([_]bool{ false, true }) |emit_hazard| { + var hazard_count: u3 = 0; + var part_disp: i32 = 0; + for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| { + defer part_disp += @intCast(dst_ty.abiSize(pt.zcu)); + const is_hazard = if (src_mcv.getReg()) |src_reg| + dst_reg.id() == src_reg.id() + else if (src_info) |info| + dst_reg.id() == info.addr_reg.id() + else + false; + if (is_hazard) hazard_count += 1; + if (is_hazard != emit_hazard) continue; + try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) { + .undef => if (opts.safety and part_i > 0) .{ .register = dst_regs[0] } else .undef, + dst_tag => |src_regs| .{ .register = src_regs[part_i] }, + .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), + .load_nav, + .load_uav, + .load_lazy_sym, + .load_extern_func, + => .{ .indirect = .{ + .reg = src_info.?.addr_reg, + .off = part_disp, + } }, + else => unreachable, + }, opts); + } + switch (hazard_count) { + 0 => break, + 1 => continue, + else => unreachable, + } + } + }, + .indirect => |reg_off| try self.genSetMem( + .{ .reg = reg_off.reg }, + reg_off.off, + ty, + src_mcv, + opts, + ), + .memory => { + switch (dst_mcv) { + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| + return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts), + .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {}, + else => unreachable, + } + + const addr_reg = try self.copyToTmpRegister(.usize, dst_mcv.address()); + const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_lock); + + try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv, opts); + }, + .load_frame => |frame_addr| try self.genSetMem( + .{ .frame = frame_addr.index }, + frame_addr.off, + ty, + src_mcv, + opts, + ), + .load_nav => |nav| try self.genSetMem(.{ .nav = nav }, 0, ty, src_mcv, opts), + .load_uav => |uav| try self.genSetMem(.{ .uav = uav }, 0, ty, src_mcv, opts), + .load_lazy_sym => |lazy_sym| try self.genSetMem(.{ .lazy_sym = lazy_sym }, 0, ty, src_mcv, opts), + .load_extern_func => |extern_func| try self.genSetMem(.{ .extern_func = extern_func }, 0, ty, src_mcv, opts), + } +} + +fn genSetReg( + self: *CodeGen, + dst_reg: Register, + ty: Type, + src_mcv: MCValue, + opts: CopyOptions, +) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const dst_alias = registerAlias(dst_reg, abi_size); + if (ty.bitSize(zcu) > dst_alias.size().bitSize(self.target)) + return self.fail("genSetReg called with a value larger than dst_reg", .{}); + switch (src_mcv) { + .none, + .unreach, + .dead, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + => unreachable, + .undef => if (opts.safety) switch (dst_reg.class()) { + .general_purpose, .gphi => switch (abi_size) { + 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xaa)), + 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xaaaa)), + 3...4 => try self.asmRegisterImmediate( + .{ ._, .mov }, + dst_reg.to32(), + .s(@as(i32, @bitCast(@as(u32, 0xaaaaaaaa)))), + ), + 5...8 => try self.asmRegisterImmediate( + .{ ._, .mov }, + dst_reg.to64(), + .u(0xaaaaaaaaaaaaaaaa), + ), + else => unreachable, + }, + .segment, .mmx, .sse => { + const full_ty = try pt.vectorType(.{ + .len = self.vectorSize(.float), + .child = .u8_type, + }); + try self.genSetReg(dst_reg, full_ty, try self.lowerValue( + try pt.aggregateSplatValue(full_ty, try pt.intValue(.u8, 0xaa)), + ), opts); + }, + .x87 => try self.genSetReg(dst_reg, .f80, try self.lowerValue( + try pt.floatValue(.f80, @as(f80, @bitCast(@as(u80, 0xaaaaaaaaaaaaaaaaaaaa)))), + ), opts), + .ip, .cr, .dr => unreachable, + }, + .eflags => |cc| try self.asmSetccRegister(cc, dst_reg.to8()), + .immediate => |imm| { + if (imm == 0) { + // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit + // register is the fastest way to zero a register. + try self.spillEflagsIfOccupied(); + try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); + } else if (abi_size > 4 and std.math.cast(u32, imm) != null) { + // 32-bit moves zero-extend to 64-bit. + try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm)); + } else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) { + try self.asmRegisterImmediate( + .{ ._, .mov }, + dst_alias, + .s(@intCast(@as(i64, @bitCast(imm)))), + ); + } else { + try self.asmRegisterImmediate( + .{ ._, .mov }, + dst_alias, + .u(imm), + ); + } + }, + .register => |src_reg| if (dst_reg.id() != src_reg.id()) switch (dst_reg.class()) { + .general_purpose => switch (src_reg.class()) { + .general_purpose => try self.asmRegisterRegister( + .{ ._, .mov }, + dst_alias, + registerAlias(src_reg, abi_size), + ), + .gphi => if (dst_reg.isClass(.gphi)) try self.asmRegisterRegister( + .{ ._, .mov }, + dst_alias, + registerAlias(src_reg, abi_size), + ) else { + const src_lock = self.register_manager.lockReg(src_reg); + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gphi); + + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg.to8(), src_reg); + try self.asmRegisterRegister(.{ ._, .mov }, dst_alias, tmp_reg.to8()); + }, + .segment => try self.asmRegisterRegister( + .{ ._, .mov }, + dst_alias, + src_reg, + ), + .x87, .mmx, .ip, .cr, .dr => unreachable, + .sse => if (self.hasFeature(.sse2)) try self.asmRegisterRegister( + switch (abi_size) { + 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, + 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, + else => unreachable, + }, + registerAlias(dst_reg, @max(abi_size, 4)), + src_reg.to128(), + ) else { + const frame_size = std.math.ceilPowerOfTwoAssert(u32, @max(abi_size, 4)); + const frame_index = try self.allocFrameIndex(.init(.{ + .size = frame_size, + .alignment = .fromNonzeroByteUnits(frame_size), + })); + try self.asmMemoryRegister(switch (frame_size) { + 4 => .{ ._ss, .mov }, + 8 => .{ ._ps, .movl }, + 16 => .{ ._ps, .mov }, + else => unreachable, + }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, + }, src_reg.to128()); + try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, + }); + }, + }, + .gphi => switch (src_reg.class()) { + .general_purpose => if (src_reg.isClass(.gphi)) try self.asmRegisterRegister( + .{ ._, .mov }, + dst_alias, + registerAlias(src_reg, abi_size), + ) else { + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gphi); + + try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg.to8(), src_reg.to8()); + try self.asmRegisterRegister(.{ ._, .mov }, dst_reg, tmp_reg.to8()); + }, + .gphi => try self.asmRegisterRegister( + .{ ._, .mov }, + dst_alias, + registerAlias(src_reg, abi_size), + ), + .segment, .x87, .mmx, .ip, .cr, .dr, .sse => unreachable, + }, + .segment => try self.asmRegisterRegister( + .{ ._, .mov }, + dst_reg, + switch (src_reg.class()) { + .general_purpose, .gphi, .segment => registerAlias(src_reg, abi_size), + .x87, .mmx, .ip, .cr, .dr => unreachable, + .sse => try self.copyToTmpRegister(ty, src_mcv), + }, + ), + .x87 => switch (src_reg.class()) { + .general_purpose, .gphi, .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87 => switch (src_reg) { + .st0 => try self.asmRegister(.{ .f_, .st }, dst_reg), + .st1, .st2, .st3, .st4, .st5, .st6 => switch (dst_reg) { + .st0 => { + try self.asmRegister(.{ .f_p, .st }, .st0); + try self.asmRegister(.{ .f_, .ld }, @enumFromInt(@intFromEnum(src_reg) - 1)); + }, + .st2, .st3, .st4, .st5, .st6 => if (self.register_manager.isKnownRegFree(.st7)) { + try self.asmRegister(.{ .f_, .ld }, src_reg); + try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1)); + } else { + try self.asmRegister(.{ .f_, .xch }, src_reg); + try self.asmRegister(.{ .f_, .xch }, dst_reg); + try self.asmRegister(.{ .f_, .xch }, src_reg); + }, + .st7 => { + if (!self.register_manager.isKnownRegFree(.st7)) try self.asmRegister(.{ .f_, .free }, dst_reg); + try self.asmRegister(.{ .f_, .ld }, src_reg); + try self.asmOpOnly(.{ .f_cstp, .in }); + }, + else => unreachable, + }, + else => unreachable, + }, + .sse => if (abi_size <= 16) { + const frame_index = try self.allocFrameIndex(.init(.{ + .size = 16, + .alignment = .@"16", + })); + try self.asmMemoryRegister(if (self.hasFeature(.avx)) + .{ .v_dqa, .mov } + else if (self.hasFeature(.sse2)) + .{ ._dqa, .mov } + else + .{ ._ps, .mova }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .xword } }, + }, src_reg.to128()); + try MoveStrategy.read(.load_store_x87, self, dst_reg, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .tbyte } }, + }); + } else unreachable, + }, + .mmx => unreachable, + .sse => switch (src_reg.class()) { + .general_purpose, .gphi => if (self.hasFeature(.sse2)) try self.asmRegisterRegister( + switch (abi_size) { + 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, + 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, + else => unreachable, + }, + dst_reg.to128(), + registerAlias(src_reg, @max(abi_size, 4)), + ) else { + const frame_size = std.math.ceilPowerOfTwoAssert(u32, @max(abi_size, 4)); + const frame_index = try self.allocFrameIndex(.init(.{ + .size = frame_size, + .alignment = .fromNonzeroByteUnits(frame_size), + })); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, + }, registerAlias(src_reg, abi_size)); + switch (frame_size) { + else => {}, + 8 => try self.asmRegisterRegister(.{ ._ps, .xor }, dst_reg.to128(), dst_reg.to128()), + } + try self.asmRegisterMemory(switch (frame_size) { + 4 => .{ ._ss, .mov }, + 8 => .{ ._ps, .movl }, + 16 => .{ ._ps, .mova }, + else => unreachable, + }, dst_reg.to128(), .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, + }); + }, + .segment => try self.genSetReg( + dst_reg, + ty, + .{ .register = try self.copyToTmpRegister(ty, src_mcv) }, + opts, + ), + .x87 => if (abi_size <= 16) { + const frame_index = try self.allocFrameIndex(.init(.{ + .size = 16, + .alignment = .@"16", + })); + try MoveStrategy.write(.load_store_x87, self, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .tbyte } }, + }, src_reg); + try self.asmRegisterMemory(if (self.hasFeature(.avx)) + .{ .v_dqa, .mov } + else if (self.hasFeature(.sse2)) + .{ ._dqa, .mov } + else + .{ ._ps, .mova }, dst_reg.to128(), .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .xword } }, + }); + } else unreachable, + .mmx, .ip, .cr, .dr => unreachable, + .sse => try self.asmRegisterRegister( + @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) { + else => switch (abi_size) { + 1...16 => if (self.hasFeature(.avx)) + .{ .v_dqa, .mov } + else if (self.hasFeature(.sse2)) + .{ ._dqa, .mov } + else + .{ ._ps, .mova }, + 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, + else => null, + }, + .float => switch (ty.scalarType(zcu).floatBits(self.target)) { + 16, 128 => switch (abi_size) { + 2...16 => if (self.hasFeature(.avx)) + .{ .v_dqa, .mov } + else if (self.hasFeature(.sse2)) + .{ ._dqa, .mov } + else + .{ ._ps, .mova }, + 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, + else => null, + }, + 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova }, + 64 => if (self.hasFeature(.avx)) + .{ .v_pd, .mova } + else if (self.hasFeature(.sse2)) + .{ ._pd, .mova } + else + .{ ._ps, .mova }, + 80 => null, + else => unreachable, + }, + }) orelse return self.fail("TODO implement genSetReg for {f}", .{ty.fmt(pt)}), + dst_alias, + registerAlias(src_reg, abi_size), + ), + }, + .ip, .cr, .dr => unreachable, + } else if ((dst_reg.class() == .gphi) != (src_reg.class() == .gphi)) try self.asmRegisterRegister( + .{ ._, .mov }, + dst_reg.to8(), + src_reg.to8(), + ), + inline .register_pair, + .register_triple, + .register_quadruple, + => |src_regs| switch (dst_reg.class()) { + .general_purpose => switch (src_regs[0].class()) { + .general_purpose => try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts), + else => unreachable, + }, + .sse => switch (src_regs[0].class()) { + .general_purpose => if (abi_size <= 16) { + if (self.hasFeature(.avx)) { + try self.asmRegisterRegister(.{ .v_q, .mov }, dst_reg.to128(), src_regs[0].to64()); + try self.asmRegisterRegisterRegisterImmediate( + .{ .vp_q, .insr }, + dst_reg.to128(), + dst_reg.to128(), + src_regs[1].to64(), + .u(1), + ); + } else if (self.hasFeature(.sse4_1)) { + try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64()); + try self.asmRegisterRegisterImmediate(.{ .p_q, .insr }, dst_reg.to128(), src_regs[1].to64(), .u(1)); + } else if (self.hasFeature(.sse2)) { + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterRegister(.{ ._q, .mov }, dst_reg.to128(), src_regs[0].to64()); + try self.asmRegisterRegister(.{ ._q, .mov }, tmp_reg.to128(), src_regs[1].to64()); + try self.asmRegisterRegister(.{ ._ps, .movlh }, dst_reg.to128(), tmp_reg.to128()); + } else { + const frame_index = try self.allocFrameIndex(.init(.{ + .size = 16, + .alignment = .@"16", + })); + for (src_regs, 0..) |src_reg, src_index| try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .qword, .disp = @intCast(8 * src_index) } }, + }, src_reg.to64()); + try self.asmRegisterMemory(.{ ._ps, .mova }, dst_reg.to128(), .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .xword } }, + }); + } + } else unreachable, + else => unreachable, + }, + else => unreachable, + }, + .register_offset, + .indirect, + .load_frame, + .lea_frame, + => try @as(MoveStrategy, switch (src_mcv) { + .register_offset => |reg_off| switch (reg_off.off) { + 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }, opts), + else => .{ .load_store = .{ ._, .lea } }, + }, + .indirect => try self.moveStrategy(ty, dst_reg.class(), false), + .load_frame => |frame_addr| try self.moveStrategy( + ty, + dst_reg.class(), + self.getFrameAddrAlignment(frame_addr).compare(.gte, .fromLog2Units( + std.math.log2_int_ceil(u64, @divExact(dst_reg.size().bitSize(self.target), 8)), + )), + ), + .lea_frame => .{ .load_store = .{ ._, .lea } }, + else => unreachable, + }).read(self, dst_alias, switch (src_mcv) { + .register_offset, .indirect => |reg_off| .{ + .base = .{ .reg = reg_off.reg.to64() }, + .mod = .{ .rm = .{ + .size = self.memSize(ty), + .disp = reg_off.off, + } }, + }, + .load_frame, .lea_frame => |frame_addr| .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = self.memSize(ty), + .disp = frame_addr.off, + } }, + }, + else => unreachable, + }), + .register_overflow => |src_reg_ov| { + const ip = &zcu.intern_pool; + const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(ty.toIntern())) { + .tuple_type => |tuple_type| { + const tuple_field_types = tuple_type.types.get(ip); + assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type); + break :first_ty tuple_field_types[0]; + }, + .opt_type => |opt_child| { + assert(!ty.optionalReprIsPayload(zcu)); + break :first_ty opt_child; + }, + else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, ty.fmt(pt) }), + }); + const first_size: u31 = @intCast(first_ty.abiSize(zcu)); + const frame_size = std.math.ceilPowerOfTwoAssert(u32, abi_size); + const frame_index = try self.allocFrameIndex(.init(.{ + .size = frame_size, + .alignment = .fromNonzeroByteUnits(frame_size), + })); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(first_size) } }, + }, registerAlias(src_reg_ov.reg, first_size)); + try self.asmSetccMemory(src_reg_ov.eflags, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .byte, .disp = first_size } }, + }); + try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, + }); + }, + .register_mask => |src_reg_mask| { + assert(src_reg_mask.reg.isClass(.sse)); + const has_avx = self.hasFeature(.avx); + const bits_reg = switch (dst_reg.class()) { + .general_purpose => dst_reg, + else => try self.register_manager.allocReg(null, abi.RegisterClass.gp), + }; + const bits_lock = self.register_manager.lockReg(bits_reg); + defer if (bits_lock) |lock| self.register_manager.unlockReg(lock); + + const pack_reg = switch (src_reg_mask.info.scalar) { + else => src_reg_mask.reg, + .word => try self.register_manager.allocReg(null, abi.RegisterClass.sse), + }; + const pack_lock = self.register_manager.lockReg(pack_reg); + defer if (pack_lock) |lock| self.register_manager.unlockReg(lock); + + var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(self.target), 8)); + switch (src_reg_mask.info.scalar) { + else => {}, + .word => { + const src_alias = registerAlias(src_reg_mask.reg, mask_size); + const pack_alias = registerAlias(pack_reg, mask_size); + if (has_avx) { + try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias); + } else { + try self.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias); + try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias); + } + mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable; + }, + } + try self.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) { + .byte, .word => if (has_avx) .vp_b else .p_b, + .dword => if (has_avx) .v_ps else ._ps, + .qword => if (has_avx) .v_pd else ._pd, + else => unreachable, + }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size)); + if (src_reg_mask.info.inverted) try self.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size)); + try self.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{}); + }, + .memory, .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => { + switch (src_mcv) { + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| + return (try self.moveStrategy( + ty, + dst_reg.class(), + ty.abiAlignment(zcu).check(@as(u32, @bitCast(small_addr))), + )).read(self, dst_alias, .{ + .base = .{ .reg = .ds }, + .mod = .{ .rm = .{ + .size = self.memSize(ty), + .disp = small_addr, + } }, + }), + .load_nav => |nav| switch (dst_reg.class()) { + .general_purpose, .gphi => { + try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ + .base = .{ .nav = nav }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }); + return; + }, + .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87, .sse => {}, + }, + .load_uav => |uav| switch (dst_reg.class()) { + .general_purpose, .gphi => { + try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ + .base = .{ .uav = uav }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }); + return; + }, + .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87, .sse => {}, + }, + .load_lazy_sym => |lazy_sym| switch (dst_reg.class()) { + .general_purpose, .gphi => { + try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ + .base = .{ .lazy_sym = lazy_sym }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }); + return; + }, + .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87, .sse => {}, + }, + .load_extern_func => |extern_func| switch (dst_reg.class()) { + .general_purpose, .gphi => { + try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ + .base = .{ .extern_func = extern_func }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }); + return; + }, + .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87, .sse => {}, + }, + else => unreachable, + } + + const addr_reg = try self.copyToTmpRegister(.usize, src_mcv.address()); + const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_lock); + + try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, dst_alias, .{ + .base = .{ .reg = addr_reg.to64() }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }); + }, + .lea_nav => |nav| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ + .base = .{ .nav = nav }, + }), + .lea_uav => |uav| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ + .base = .{ .uav = uav }, + }), + .lea_lazy_sym => |lazy_sym| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ + .base = .{ .lazy_sym = lazy_sym }, + }), + .lea_extern_func => |lazy_sym| try self.asmRegisterMemory(.{ ._, .lea }, dst_reg.to64(), .{ + .base = .{ .extern_func = lazy_sym }, + }), + .air_ref => |src_ref| try self.genSetReg(dst_reg, ty, try self.resolveInst(src_ref), opts), + } +} + +fn genSetMem( + self: *CodeGen, + base: Memory.Base, + disp: i32, + ty: Type, + src_mcv: MCValue, + opts: CopyOptions, +) InnerError!void { + const pt = self.pt; + const zcu = pt.zcu; + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const dst_ptr_mcv: MCValue = switch (base) { + .none => .{ .immediate = @bitCast(@as(i64, disp)) }, + .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, + .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, + .table, .rip_inst, .lazy_sym => unreachable, + .nav => |nav| .{ .lea_nav = nav }, + .uav => |uav| .{ .lea_uav = uav }, + .extern_func => |extern_func| .{ .lea_extern_func = extern_func }, + }; + switch (src_mcv) { + .none, + .unreach, + .dead, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + => unreachable, + .undef => if (opts.safety) try self.genInlineMemset( + dst_ptr_mcv, + src_mcv, + .{ .immediate = abi_size }, + opts, + ), + .immediate => |imm| switch (abi_size) { + 1, 2, 4 => { + const immediate: Immediate = switch (if (ty.isAbiInt(zcu)) + ty.intInfo(zcu).signedness + else + .unsigned) { + .signed => .s(@truncate(@as(i64, @bitCast(imm)))), + .unsigned => .u(@as(u32, @intCast(imm))), + }; + try self.asmMemoryImmediate( + .{ ._, .mov }, + .{ .base = base, .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = disp, + } } }, + immediate, + ); + }, + 3, 5...7 => unreachable, + else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| { + try self.asmMemoryImmediate( + .{ ._, .mov }, + .{ .base = base, .mod = .{ .rm = .{ + .size = .fromSize(abi_size), + .disp = disp, + } } }, + .s(small), + ); + } else { + var offset: i32 = 0; + while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate( + .{ ._, .mov }, + .{ .base = base, .mod = .{ .rm = .{ + .size = .dword, + .disp = disp + offset, + } } }, + if (ty.isSignedInt(zcu)) .s( + @truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)), + ) else .u( + @as(u32, @truncate(if (std.math.cast(u6, offset * 8)) |shift| imm >> shift else 0)), + ), + ); + }, + }, + .eflags => |cc| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{ + .rm = .{ .size = .byte, .disp = disp }, + } }), + .register => |src_reg| { + const ip = &zcu.intern_pool; + const mem_size = switch (base) { + .frame => |base_fi| mem_size: { + assert(disp >= 0); + const frame_abi_size = self.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)]; + const frame_spill_pad = self.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)]; + assert(frame_abi_size - frame_spill_pad - disp >= abi_size); + break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size) frame_abi_size else abi_size; + }, + else => abi_size, + }; + const src_alias = registerAlias(src_reg, @intCast(self.unalignedSize(ty))); + const src_class = src_alias.class(); + const src_size: Memory.Size = switch (src_class) { + .general_purpose, .gphi, .segment, .ip, .cr, .dr => src_alias.size(), + .mmx, .sse => .fromSize(abi_size), + .x87 => switch (abi.classifySystemV(ty, zcu, self.target, .other)[0]) { + else => unreachable, + .float => .dword, + .float_combine, .sse => .qword, + .x87 => .tbyte, + }, + }; + const src_bit_size = src_size.bitSize(self.target); + const src_align: InternPool.Alignment = .fromNonzeroByteUnits(std.math.ceilPowerOfTwoAssert(u64, src_bit_size)); + const src_byte_size = @divExact(src_bit_size, 8); + if (src_byte_size > mem_size) { + const frame_index = try self.allocFrameIndex(.init(.{ + .size = src_byte_size, + .alignment = src_align, + })); + const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } }; + try (try self.moveStrategy(ty, src_class, true)).write(self, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = src_size } }, + }, src_alias); + try self.genSetMem(base, disp, ty, frame_mcv, opts); + try self.freeValue(frame_mcv, .{}); + } else try (try self.moveStrategy(ty, src_class, switch (base) { + .none => src_align.check(@as(u32, @bitCast(disp))), + .reg => |reg| switch (reg) { + .es, .cs, .ss, .ds => src_align.check(@as(u32, @bitCast(disp))), + else => false, + }, + .frame => |frame_index| self.getFrameAddrAlignment(.{ + .index = frame_index, + .off = disp, + }).compare(.gte, src_align), + .table, .rip_inst, .lazy_sym, .extern_func => unreachable, + .nav => |nav| ip.getNav(nav).getAlignment().compare(.gte, src_align), + .uav => |uav| Type.fromInterned(uav.orig_ty).ptrAlignment(zcu).compare(.gte, src_align), + })).write(self, .{ + .base = base, + .mod = .{ .rm = .{ .size = src_size, .disp = disp } }, + }, src_alias); + }, + inline .register_pair, .register_triple, .register_quadruple => |src_regs| { + var part_disp: i32 = disp; + for (try self.splitType(src_regs.len, ty), src_regs) |src_ty, src_reg| { + try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts); + part_disp += @intCast(src_ty.abiSize(zcu)); + } + }, + .register_overflow => |ro| switch (ty.zigTypeTag(zcu)) { + .@"struct" => { + try self.genSetMem( + base, + disp + @as(i32, @intCast(ty.structFieldOffset(0, zcu))), + ty.fieldType(0, zcu), + .{ .register = ro.reg }, + opts, + ); + try self.genSetMem( + base, + disp + @as(i32, @intCast(ty.structFieldOffset(1, zcu))), + ty.fieldType(1, zcu), + .{ .eflags = ro.eflags }, + opts, + ); + }, + .optional => { + assert(!ty.optionalReprIsPayload(zcu)); + const child_ty = ty.optionalChild(zcu); + try self.genSetMem(base, disp, child_ty, .{ .register = ro.reg }, opts); + try self.genSetMem( + base, + disp + @as(i32, @intCast(child_ty.abiSize(zcu))), + .bool, + .{ .eflags = ro.eflags }, + opts, + ); + }, + else => return self.fail("TODO implement genSetMem for {s} of {f}", .{ + @tagName(src_mcv), ty.fmt(pt), + }), + }, + .register_offset => |reg_off| { + const src_reg = self.copyToTmpRegister(ty, src_mcv) catch |err| switch (err) { + error.OutOfRegisters => { + const src_reg = registerAlias(reg_off.reg, abi_size); + try self.asmRegisterMemory(.{ ._, .lea }, src_reg, .{ + .base = .{ .reg = src_reg }, + .mod = .{ .rm = .{ .disp = reg_off.off } }, + }); + try self.genSetMem(base, disp, ty, .{ .register = reg_off.reg }, opts); + return self.asmRegisterMemory(.{ ._, .lea }, src_reg, .{ + .base = .{ .reg = src_reg }, + .mod = .{ .rm = .{ .disp = -reg_off.off } }, + }); + }, + else => |e| return e, + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); + }, + .register_mask => { + const src_reg = try self.copyToTmpRegister(ty, src_mcv); + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); + }, + .memory, + .indirect, + .load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => switch (abi_size) { + 0 => {}, + 1, 2, 4, 8 => { + const src_reg = try self.copyToTmpRegister(ty, src_mcv); + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); + }, + else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }, .{ .no_alias = true }), + }, + .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts), + } +} + +fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct { + no_alias: bool, +}) InnerError!void { + if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) { + else => {}, + .immediate => |len_imm| switch (len_imm) { + else => {}, + 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { + try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte })); + try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8()); + return; + }, + 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { + try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word })); + try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16()); + return; + }, + 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { + try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword })); + try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32()); + return; + }, + 8 => if (self.target.cpu.arch == .x86_64) { + if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { + try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword })); + try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64()); + return; + } + }, + 16 => if (self.hasFeature(.avx)) { + if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { + try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); + try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); + return; + } + } else if (self.hasFeature(.sse2)) { + if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { + try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); + try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); + return; + } + } else if (self.hasFeature(.sse)) { + if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { + try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); + try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); + return; + } + }, + 32 => if (self.hasFeature(.avx)) { + if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { + try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword })); + try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256()); + return; + } + }, + }, + }; + try self.spillRegisters(&.{ .rsi, .rdi, .rcx }); + try self.genSetReg(.rsi, .usize, src_ptr, .{}); + try self.genSetReg(.rdi, .usize, dst_ptr, .{}); + try self.genSetReg(.rcx, .usize, len, .{}); + try self.asmOpOnly(.{ .@"rep _sb", .mov }); +} + +fn genInlineMemset( + self: *CodeGen, + dst_ptr: MCValue, + value: MCValue, + len: MCValue, + opts: CopyOptions, +) InnerError!void { + try self.spillRegisters(&.{ .rdi, .al, .rcx }); + try self.genSetReg(.rdi, .usize, dst_ptr, .{}); + try self.genSetReg(.al, .u8, value, opts); + try self.genSetReg(.rcx, .usize, len, .{}); + try self.asmOpOnly(.{ .@"rep _sb", .sto }); +} + +fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const dst_ty = self.typeOfIndex(inst); + const src_ty = self.typeOf(ty_op.operand); + + const result = result: { + const src_mcv = try self.resolveInst(ty_op.operand); + if (dst_ty.isPtrAtRuntime(zcu) and src_ty.isPtrAtRuntime(zcu)) switch (src_mcv) { + .lea_frame => break :result src_mcv, + else => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv, + }; + + const dst_rc = self.regSetForType(dst_ty); + const src_rc = self.regSetForType(src_ty); + + const src_lock = if (src_mcv.getReg()) |src_reg| self.register_manager.lockReg(src_reg) else null; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = if (src_mcv != .register_mask and + (if (src_mcv.getReg()) |src_reg| src_reg.isClass(.general_purpose) else true) and + dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and + dst_ty.abiAlignment(zcu).order(src_ty.abiAlignment(zcu)).compare(.lte) and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(switch (src_mcv) { + else => switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { + .lt => dst_ty, + .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty, + .gt => src_ty, + }, + .register_mask => src_ty, + }, dst_mcv, src_mcv, .{}); + break :dst dst_mcv; + }; + + if (dst_ty.isRuntimeFloat()) break :result dst_mcv; + + if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and + dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv; + + const abi_size = dst_ty.abiSize(zcu); + const bit_size = dst_ty.bitSize(zcu); + if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv; + + const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable; + const high_mcv: MCValue = switch (dst_mcv) { + .register => |dst_reg| .{ .register = dst_reg }, + .register_pair => |dst_regs| .{ .register = dst_regs[1] }, + else => dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(), + }; + const high_reg = if (high_mcv.isRegister()) + high_mcv.getReg().? + else + try self.copyToTmpRegister(.usize, high_mcv); + const high_lock = self.register_manager.lockReg(high_reg); + defer if (high_lock) |lock| self.register_manager.unlockReg(lock); + try self.truncateRegister(dst_ty, high_reg); + if (!high_mcv.isRegister()) try self.genCopy( + if (abi_size <= 8) dst_ty else .usize, + high_mcv, + .{ .register = high_reg }, + .{}, + ); + var offset = dst_limbs_len * 8; + if (offset < abi_size) { + const dst_signedness: std.builtin.Signedness = if (dst_ty.isAbiInt(zcu)) + dst_ty.intInfo(zcu).signedness + else + .unsigned; + const ext_mcv: MCValue = ext_mcv: switch (dst_signedness) { + .signed => { + try self.asmRegisterImmediate(.{ ._r, .sa }, high_reg, .u(63)); + break :ext_mcv .{ .register = high_reg }; + }, + .unsigned => .{ .immediate = 0 }, + }; + while (offset < abi_size) : (offset += 8) { + const limb_mcv: MCValue = switch (dst_mcv) { + .register => |dst_reg| .{ .register = dst_reg }, + .register_pair => |dst_regs| .{ .register = dst_regs[@divExact(offset, 8)] }, + else => dst_mcv.address().offset(offset).deref(), + }; + const limb_lock = if (limb_mcv.isRegister()) + self.register_manager.lockReg(limb_mcv.getReg().?) + else + null; + defer if (limb_lock) |lock| self.register_manager.unlockReg(lock); + try self.genCopy(.usize, limb_mcv, ext_mcv, .{}); + } + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const slice_ty = self.typeOfIndex(inst); + const ptr_ty = self.typeOf(ty_op.operand); + const ptr = try self.resolveInst(ty_op.operand); + const array_ty = ptr_ty.childType(zcu); + const array_len = array_ty.arrayLen(zcu); + + const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu)); + try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{}); + try self.genSetMem( + .{ .frame = frame_index }, + @intCast(ptr_ty.abiSize(zcu)), + .usize, + .{ .immediate = array_len }, + .{}, + ); + + const result = MCValue{ .load_frame = .{ .index = frame_index } }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dst_ty = self.typeOfIndex(inst); + const dst_bits = dst_ty.floatBits(self.target); + + const src_ty = self.typeOf(ty_op.operand); + const src_bits: u32 = @intCast(src_ty.bitSize(zcu)); + const src_signedness = + if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned; + const src_size = std.math.divCeil(u32, @max(switch (src_signedness) { + .signed => src_bits, + .unsigned => src_bits + 1, + }, 32), 8) catch unreachable; + + const result = result: { + if (switch (dst_bits) { + 16, 80, 128 => true, + 32, 64 => src_size > 8, + else => unreachable, + }) { + if (src_bits > 128) return self.fail("TODO implement airFloatFromInt from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + + var sym_buf: ["__floatun?i?f".len]u8 = undefined; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = dst_ty.toIntern(), + .param_types = &.{src_ty.toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "__float{s}{c}i{c}f", .{ + switch (src_signedness) { + .signed => "", + .unsigned => "un", + }, + intCompilerRtAbiName(src_bits), + floatCompilerRtAbiName(dst_bits), + }) catch unreachable, + } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); + } + + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv); + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg); + + const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty)); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_ty.zigTypeTag(zcu)) { + .float => switch (dst_ty.floatBits(self.target)) { + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .cvtsi2 } else .{ ._ss, .cvtsi2 }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .cvtsi2 } else .{ ._sd, .cvtsi2 }, + 16, 80, 128 => null, + else => unreachable, + }, + else => null, + }) orelse return self.fail("TODO implement airFloatFromInt from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + const dst_alias = dst_reg.to128(); + const src_alias = registerAlias(src_reg, src_size); + switch (mir_tag[0]) { + .v_ss, .v_sd => try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, src_alias), + else => try self.asmRegisterRegister(mir_tag, dst_alias, src_alias), + } + + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const dst_ty = self.typeOfIndex(inst); + const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu)); + const dst_signedness = + if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned; + const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) { + .signed => dst_bits, + .unsigned => dst_bits + 1, + }, 32), 8) catch unreachable; + + const src_ty = self.typeOf(ty_op.operand); + const src_bits = src_ty.floatBits(self.target); + + const result = result: { + if (switch (src_bits) { + 16, 80, 128 => true, + 32, 64 => dst_size > 8, + else => unreachable, + }) { + if (dst_bits > 128) return self.fail("TODO implement airIntFromFloat from {f} to {f}", .{ + src_ty.fmt(pt), dst_ty.fmt(pt), + }); + + var sym_buf: ["__fixuns?f?i".len]u8 = undefined; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = dst_ty.toIntern(), + .param_types = &.{src_ty.toIntern()}, + .sym = std.fmt.bufPrint(&sym_buf, "__fix{s}{c}f{c}i", .{ + switch (dst_signedness) { + .signed => "", + .unsigned => "uns", + }, + floatCompilerRtAbiName(src_bits), + intCompilerRtAbiName(dst_bits), + }) catch unreachable, + } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{}); + } + + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(src_ty, src_mcv); + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty)); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + try self.asmRegisterRegister( + switch (src_bits) { + 32 => if (self.hasFeature(.avx)) .{ .v_, .cvttss2si } else .{ ._, .cvttss2si }, + 64 => if (self.hasFeature(.avx)) .{ .v_, .cvttsd2si } else .{ ._, .cvttsd2si }, + else => unreachable, + }, + registerAlias(dst_reg, dst_size), + src_reg.to128(), + ); + + if (dst_bits < dst_size * 8) try self.truncateRegister(dst_ty, dst_reg); + + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + + const dst_ty = self.typeOfIndex(inst); + const ptr_ty = self.typeOf(extra.ptr); + const val_ty = self.typeOf(extra.expected_value); + const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu)); + + try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx }); + const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx }); + defer for (regs_lock) |lock| self.register_manager.unlockReg(lock); + + const exp_mcv = try self.resolveInst(extra.expected_value); + if (val_abi_size > 8) { + const exp_addr_mcv: MCValue = switch (exp_mcv) { + .memory, .indirect, .load_frame => exp_mcv.address(), + else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) }, + }; + const exp_addr_lock = + if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{}); + try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{}); + } else try self.genSetReg(.rax, val_ty, exp_mcv, .{}); + + const new_mcv = try self.resolveInst(extra.new_value); + const new_reg = if (val_abi_size > 8) new: { + const new_addr_mcv: MCValue = switch (new_mcv) { + .memory, .indirect, .load_frame => new_mcv.address(), + else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) }, + }; + const new_addr_lock = + if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; + defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{}); + try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{}); + break :new null; + } else try self.copyToTmpRegister(val_ty, new_mcv); + const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; + defer if (new_lock) |lock| self.register_manager.unlockReg(lock); + + const ptr_mcv = try self.resolveInst(extra.ptr); + const mem_size: Memory.Size = .fromSize(val_abi_size); + const ptr_mem: Memory = switch (ptr_mcv) { + .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }), + else => .{ + .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, + .mod = .{ .rm = .{ .size = mem_size } }, + }, + }; + switch (ptr_mem.mod) { + .rm => {}, + .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}), + } + const ptr_lock = switch (ptr_mem.base) { + .none, .frame, .nav, .uav => null, + .reg => |reg| self.register_manager.lockReg(reg), + .table, .rip_inst, .lazy_sym, .extern_func => unreachable, + }; + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + try self.spillEflagsIfOccupied(); + const null_reg = if (dst_ty.optionalReprIsPayload(zcu) and !self.liveness.isUnused(inst)) null_reg: { + const null_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + try self.asmRegisterRegister(.{ ._, .xor }, null_reg.to32(), null_reg.to32()); + break :null_reg null_reg; + } else null; + const null_lock = if (null_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; + defer if (null_lock) |lock| self.register_manager.unlockReg(lock); + + if (val_abi_size <= 8) try self.asmMemoryRegister( + .{ .@"lock _", .cmpxchg }, + ptr_mem, + registerAlias(new_reg.?, val_abi_size), + ) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); + + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .unreach; + + if (null_reg) |reg| try self.asmCmovccRegisterRegister( + .e, + registerAlias(.rax, val_abi_size), + registerAlias(reg, val_abi_size), + ); + + if (val_abi_size <= 8) { + if (null_reg) |_| break :result .{ .register = .rax }; + self.eflags_inst = inst; + break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } }; + } + + const dst_mcv = try self.allocRegOrMem(inst, false); + try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{}); + try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{}); + if (null_reg == null) try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{}); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); +} + +fn atomicOp( + self: *CodeGen, + ptr_mcv: MCValue, + val_mcv: MCValue, + ptr_ty: Type, + val_ty: Type, + unused: bool, + rmw_op: ?std.builtin.AtomicRmwOp, + order: std.builtin.AtomicOrder, +) InnerError!MCValue { + const pt = self.pt; + const zcu = pt.zcu; + const ptr_lock = switch (ptr_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const val_lock = switch (val_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (val_lock) |lock| self.register_manager.unlockReg(lock); + + const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu)); + const mem_size: Memory.Size = .fromSize(val_abi_size); + const ptr_mem: Memory = switch (ptr_mcv) { + .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }), + else => .{ + .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, + .mod = .{ .rm = .{ .size = mem_size } }, + }, + }; + switch (ptr_mem.mod) { + .rm => {}, + .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}), + } + const mem_lock = switch (ptr_mem.base) { + .none, .frame, .nav, .uav => null, + .reg => |reg| self.register_manager.lockReg(reg), + .table, .rip_inst, .lazy_sym, .extern_func => unreachable, + }; + defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); + + const use_sse = rmw_op orelse .Xchg != .Xchg and val_ty.isRuntimeFloat(); + const strat: enum { lock, loop, libcall } = if (use_sse) .loop else switch (rmw_op orelse .Xchg) { + .Xchg, + .Add, + .Sub, + => if (val_abi_size <= 8) .lock else if (val_abi_size <= 16) .loop else .libcall, + .And, + .Or, + .Xor, + => if (val_abi_size <= 8 and unused) .lock else if (val_abi_size <= 16) .loop else .libcall, + .Nand, + .Max, + .Min, + => if (val_abi_size <= 16) .loop else .libcall, + }; + switch (strat) { + .lock => { + const mir_tag: Mir.Inst.FixedTag = if (rmw_op) |op| switch (op) { + .Xchg => if (unused) .{ ._, .mov } else .{ ._g, .xch }, + .Add => .{ .@"lock _", if (unused) .add else .xadd }, + .Sub => .{ .@"lock _", if (unused) .sub else .xadd }, + .And => .{ .@"lock _", .@"and" }, + .Or => .{ .@"lock _", .@"or" }, + .Xor => .{ .@"lock _", .xor }, + else => unreachable, + } else switch (order) { + .unordered, .monotonic, .release, .acq_rel => .{ ._, .mov }, + .acquire => unreachable, + .seq_cst => .{ ._g, .xch }, + }; + + const dst_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + try self.genSetReg(dst_reg, val_ty, val_mcv, .{}); + if (rmw_op == std.builtin.AtomicRmwOp.Sub and mir_tag[1] == .xadd) { + try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv); + } + try self.asmMemoryRegister(mir_tag, ptr_mem, registerAlias(dst_reg, val_abi_size)); + + return if (unused) .unreach else dst_mcv; + }, + .loop => _ = if (val_abi_size <= 8) { + const sse_reg: Register = if (use_sse) + try self.register_manager.allocReg(null, abi.RegisterClass.sse) + else + undefined; + const sse_lock = + if (use_sse) self.register_manager.lockRegAssumeUnused(sse_reg) else undefined; + defer if (use_sse) self.register_manager.unlockReg(sse_lock); + + const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const tmp_mcv = MCValue{ .register = tmp_reg }; + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem); + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + if (!use_sse and rmw_op orelse .Xchg != .Xchg) { + try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax }, .{}); + } + if (rmw_op) |op| if (use_sse) { + const mir_tag = @as(?Mir.Inst.FixedTag, switch (op) { + .Add => switch (val_ty.floatBits(self.target)) { + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, + else => null, + }, + .Sub => switch (val_ty.floatBits(self.target)) { + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, + else => null, + }, + .Min => switch (val_ty.floatBits(self.target)) { + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, + else => null, + }, + .Max => switch (val_ty.floatBits(self.target)) { + 32 => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, + 64 => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, + else => null, + }, + else => unreachable, + }) orelse return self.fail("TODO implement atomicOp of {s} for {f}", .{ + @tagName(op), val_ty.fmt(pt), + }); + try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{}); + switch (mir_tag[0]) { + .v_ss, .v_sd => if (val_mcv.isBase()) try self.asmRegisterRegisterMemory( + mir_tag, + sse_reg.to128(), + sse_reg.to128(), + try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), + ) else try self.asmRegisterRegisterRegister( + mir_tag, + sse_reg.to128(), + sse_reg.to128(), + (if (val_mcv.isRegister()) + val_mcv.getReg().? + else + try self.copyToTmpRegister(val_ty, val_mcv)).to128(), + ), + ._ss, ._sd => if (val_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + sse_reg.to128(), + try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), + ) else try self.asmRegisterRegister( + mir_tag, + sse_reg.to128(), + (if (val_mcv.isRegister()) + val_mcv.getReg().? + else + try self.copyToTmpRegister(val_ty, val_mcv)).to128(), + ), + else => unreachable, + } + try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg }, .{}); + } else switch (op) { + .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv, .{}), + .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv), + .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv), + .And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv), + .Nand => { + try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv); + try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv); + }, + .Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv), + .Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv), + .Min, .Max => { + const cc: Condition = switch (if (val_ty.isAbiInt(zcu)) + val_ty.intInfo(zcu).signedness + else + .unsigned) { + .unsigned => switch (op) { + .Min => .a, + .Max => .b, + else => unreachable, + }, + .signed => switch (op) { + .Min => .g, + .Max => .l, + else => unreachable, + }, + }; + + const cmov_abi_size = @max(val_abi_size, 2); + switch (val_mcv) { + .register => |val_reg| { + try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv); + try self.asmCmovccRegisterRegister( + cc, + registerAlias(tmp_reg, cmov_abi_size), + registerAlias(val_reg, cmov_abi_size), + ); + }, + .memory, .indirect, .load_frame => { + try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv); + try self.asmCmovccRegisterMemory( + cc, + registerAlias(tmp_reg, cmov_abi_size), + try val_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }), + ); + }, + else => { + const mat_reg = try self.copyToTmpRegister(val_ty, val_mcv); + const mat_lock = self.register_manager.lockRegAssumeUnused(mat_reg); + defer self.register_manager.unlockReg(mat_lock); + + try self.genBinOpMir( + .{ ._, .cmp }, + val_ty, + tmp_mcv, + .{ .register = mat_reg }, + ); + try self.asmCmovccRegisterRegister( + cc, + registerAlias(tmp_reg, cmov_abi_size), + registerAlias(mat_reg, cmov_abi_size), + ); + }, + } + }, + }; + try self.asmMemoryRegister( + .{ .@"lock _", .cmpxchg }, + ptr_mem, + registerAlias(tmp_reg, val_abi_size), + ); + _ = try self.asmJccReloc(.ne, loop); + return if (unused) .unreach else .{ .register = .rax }; + } else { + try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{ + .base = ptr_mem.base, + .mod = .{ .rm = .{ + .size = .qword, + .index = ptr_mem.mod.rm.index, + .scale = ptr_mem.mod.rm.scale, + .disp = ptr_mem.mod.rm.disp + 0, + } }, + }); + try self.asmRegisterMemory(.{ ._, .mov }, .rdx, .{ + .base = ptr_mem.base, + .mod = .{ .rm = .{ + .size = .qword, + .index = ptr_mem.mod.rm.index, + .scale = ptr_mem.mod.rm.scale, + .disp = ptr_mem.mod.rm.disp + 8, + } }, + }); + const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); + const val_mem_mcv: MCValue = switch (val_mcv) { + .memory, .indirect, .load_frame => val_mcv, + else => .{ .indirect = .{ + .reg = try self.copyToTmpRegister(.usize, val_mcv.address()), + } }, + }; + const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword }); + const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }); + if (rmw_op != std.builtin.AtomicRmwOp.Xchg) { + try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax); + try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx); + } + if (rmw_op) |op| switch (op) { + .Xchg => { + try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem); + }, + .Add => { + try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem); + }, + .Sub => { + try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem); + }, + .And => { + try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem); + }, + .Nand => { + try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem); + try self.asmRegister(.{ ._, .not }, .rbx); + try self.asmRegister(.{ ._, .not }, .rcx); + }, + .Or => { + try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem); + }, + .Xor => { + try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem); + }, + .Min, .Max => { + const cc: Condition = switch (if (val_ty.isAbiInt(zcu)) + val_ty.intInfo(zcu).signedness + else + .unsigned) { + .unsigned => switch (op) { + .Min => .a, + .Max => .b, + else => unreachable, + }, + .signed => switch (op) { + .Min => .g, + .Max => .l, + else => unreachable, + }, + }; + + const tmp_reg = try self.copyToTmpRegister(.usize, .{ .register = .rcx }); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + try self.asmRegisterMemory(.{ ._, .cmp }, .rbx, val_lo_mem); + try self.asmRegisterMemory(.{ ._, .sbb }, tmp_reg, val_hi_mem); + try self.asmCmovccRegisterMemory(cc, .rbx, val_lo_mem); + try self.asmCmovccRegisterMemory(cc, .rcx, val_hi_mem); + }, + }; + try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); + _ = try self.asmJccReloc(.ne, loop); + + if (unused) return .unreach; + const dst_mcv = try self.allocTempRegOrMem(val_ty, false); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_mcv.load_frame.off + 0, + } }, + }, .rax); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = dst_mcv.load_frame.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = dst_mcv.load_frame.off + 8, + } }, + }, .rdx); + return dst_mcv; + }, + .libcall => return self.fail("TODO implement x86 atomic libcall", .{}), + } +} + +fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; + + try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx }); + const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx }); + defer for (regs_lock) |lock| self.register_manager.unlockReg(lock); + + const unused = self.liveness.isUnused(inst); + + const ptr_ty = self.typeOf(pl_op.operand); + const ptr_mcv = try self.resolveInst(pl_op.operand); + + const val_ty = self.typeOf(extra.operand); + const val_mcv = try self.resolveInst(extra.operand); + + const result = + try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering()); + return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); +} + +fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void { + const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; + const result: MCValue = result: { + const ptr_ty = self.typeOf(atomic_load.ptr); + const ptr_mcv = try self.resolveInst(atomic_load.ptr); + const ptr_lock = switch (ptr_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const unused = self.liveness.isUnused(inst); + + const dst_mcv: MCValue = if (unused) + .{ .register = try self.register_manager.allocReg(null, self.regSetForType(ptr_ty.childType(self.pt.zcu))) } + else if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv)) + ptr_mcv + else + try self.allocRegOrMem(inst, true); + + try self.load(dst_mcv, ptr_ty, ptr_mcv); + + break :result if (unused) .unreach else dst_mcv; + }; + return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none }); +} + +fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const ptr_ty = self.typeOf(bin_op.lhs); + const ptr_mcv = try self.resolveInst(bin_op.lhs); + + const val_ty = self.typeOf(bin_op.rhs); + const val_mcv = try self.resolveInst(bin_op.rhs); + + const result = try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + result: { + if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; + + try self.spillRegisters(&.{ .rax, .rdi, .rsi, .rcx }); + const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx }); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + const dst = try self.resolveInst(bin_op.lhs); + const dst_ty = self.typeOf(bin_op.lhs); + const dst_locks: [2]?RegisterLock = switch (dst) { + .register => |dst_reg| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null }, + .register_pair => |dst_regs| .{ + self.register_manager.lockRegAssumeUnused(dst_regs[0]), + self.register_manager.lockRegAssumeUnused(dst_regs[1]), + }, + else => @splat(null), + }; + for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const src_val = try self.resolveInst(bin_op.rhs); + const elem_ty = self.typeOf(bin_op.rhs); + const src_val_lock: ?RegisterLock = switch (src_val) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock); + + const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu)); + + if (elem_abi_size == 1) { + const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) { + .slice => switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[0] }, + else => dst, + }, + .one => dst, + .c, .many => unreachable, + }; + const len: MCValue = switch (dst_ty.ptrSize(zcu)) { + .slice => switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[1] }, + else => dst.address().offset(8).deref(), + }, + .one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) }, + .c, .many => unreachable, + }; + const len_lock: ?RegisterLock = switch (len) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (len_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety }); + break :result; + } + + // Store the first element, and then rely on memcpy copying forwards. + // Length zero requires a runtime check - so we handle arrays specially + // here to elide it. + switch (dst_ty.ptrSize(zcu)) { + .slice => { + const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu); + + const dst_ptr: MCValue = switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[0] }, + else => dst, + }; + const len: MCValue = switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[1] }, + else => dst.address().offset(8).deref(), + }; + + // Used to store the number of elements for comparison. + // After comparison, updated to store number of bytes needed to copy. + const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const len_mcv: MCValue = .{ .register = len_reg }; + const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); + defer self.register_manager.unlockReg(len_lock); + + try self.genSetReg(len_reg, .usize, len, .{}); + try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); + + const skip_reloc = try self.asmJccReloc(.z, undefined); + try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety }); + + const second_elem_ptr_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; + const second_elem_ptr_lock = + self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); + defer self.register_manager.unlockReg(second_elem_ptr_lock); + + try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{ + .reg = try self.copyToTmpRegister(.usize, dst_ptr), + .off = elem_abi_size, + } }, .{}); + + try self.genBinOpMir(.{ ._, .sub }, .usize, len_mcv, .{ .immediate = 1 }); + try self.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + len_reg, + len_reg, + .s(elem_abi_size), + ); + try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv, .{ .no_alias = false }); + + self.performReloc(skip_reloc); + }, + .one => { + const elem_ptr_ty = try pt.singleMutPtrType(elem_ty); + + const len = dst_ty.childType(zcu).arrayLen(zcu); + + assert(len != 0); // prevented by Sema + try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety }); + + const second_elem_ptr_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; + const second_elem_ptr_lock = + self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); + defer self.register_manager.unlockReg(second_elem_ptr_lock); + + try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{ + .reg = try self.copyToTmpRegister(.usize, dst), + .off = elem_abi_size, + } }, .{}); + + const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; + try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy, .{ .no_alias = false }); + }, + .c, .many => unreachable, + } + } + return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); +} + +fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const vector_ty = self.typeOfIndex(inst); + const vector_len = vector_ty.vectorLen(zcu); + const dst_rc = self.regSetForType(vector_ty); + const scalar_ty = self.typeOf(ty_op.operand); + + const result: MCValue = result: { + switch (scalar_ty.zigTypeTag(zcu)) { + else => {}, + .bool => { + const regs = + try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp); + const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs); + defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); + + try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 }, .{}); + try self.genSetReg( + regs[1], + vector_ty, + .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) }, + .{}, + ); + const src_mcv = try self.resolveInst(ty_op.operand); + const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4); + try self.asmCmovccRegisterRegister( + switch (src_mcv) { + .eflags => |cc| cc, + .register => |src_reg| cc: { + try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1)); + break :cc .nz; + }, + else => cc: { + try self.asmMemoryImmediate( + .{ ._, .@"test" }, + try src_mcv.mem(self, .{ .size = .byte }), + .u(1), + ); + break :cc .nz; + }, + }, + registerAlias(regs[0], abi_size), + registerAlias(regs[1], abi_size), + ); + break :result .{ .register = regs[0] }; + }, + .int => if (self.hasFeature(.avx2)) avx2: { + const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(zcu).bits) { + else => null, + 1...8 => switch (vector_len) { + else => null, + 1...32 => .{ .vp_b, .broadcast }, + }, + 9...16 => switch (vector_len) { + else => null, + 1...16 => .{ .vp_w, .broadcast }, + }, + 17...32 => switch (vector_len) { + else => null, + 1...8 => .{ .vp_d, .broadcast }, + }, + 33...64 => switch (vector_len) { + else => null, + 1...4 => .{ .vp_q, .broadcast }, + }, + 65...128 => switch (vector_len) { + else => null, + 1...2 => .{ .v_i128, .broadcast }, + }, + }) orelse break :avx2; + + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const src_mcv = try self.resolveInst(ty_op.operand); + if (src_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), + try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }), + ) else { + if (mir_tag[0] == .v_i128) break :avx2; + try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); + try self.asmRegisterRegister( + mir_tag, + registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), + registerAlias(dst_reg, @intCast(scalar_ty.abiSize(zcu))), + ); + } + break :result .{ .register = dst_reg }; + } else { + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand }, .{}); + if (vector_len == 1) break :result .{ .register = dst_reg }; + + const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))); + const scalar_bits = scalar_ty.intInfo(zcu).bits; + if (switch (scalar_bits) { + 1...8 => true, + 9...128 => false, + else => unreachable, + }) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister( + .{ .vp_, .unpcklbw }, + dst_alias, + dst_alias, + dst_alias, + ) else try self.asmRegisterRegister( + .{ .p_, .unpcklbw }, + dst_alias, + dst_alias, + ); + if (switch (scalar_bits) { + 1...8 => vector_len > 2, + 9...16 => true, + 17...128 => false, + else => unreachable, + }) try self.asmRegisterRegisterImmediate( + .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl }, + dst_alias, + dst_alias, + .u(0b00_00_00_00), + ); + if (switch (scalar_bits) { + 1...8 => vector_len > 4, + 9...16 => vector_len > 2, + 17...64 => true, + 65...128 => false, + else => unreachable, + }) try self.asmRegisterRegisterImmediate( + .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf }, + dst_alias, + dst_alias, + .u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00), + ); + break :result .{ .register = dst_reg }; + }, + .float => switch (scalar_ty.floatBits(self.target)) { + 32 => switch (vector_len) { + 1 => { + const src_mcv = try self.resolveInst(ty_op.operand); + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); + break :result .{ .register = dst_reg }; + }, + 2...4 => { + const src_mcv = try self.resolveInst(ty_op.operand); + if (self.hasFeature(.avx)) { + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_ss, .broadcast }, + dst_reg.to128(), + try src_mcv.mem(self, .{ .size = .dword }), + ) else { + const src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(scalar_ty, src_mcv); + try self.asmRegisterRegisterRegisterImmediate( + .{ .v_ps, .shuf }, + dst_reg.to128(), + src_reg.to128(), + src_reg.to128(), + .u(0), + ); + } + break :result .{ .register = dst_reg }; + } else { + const dst_mcv = if (src_mcv.isRegister() and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, scalar_ty, src_mcv); + const dst_reg = dst_mcv.getReg().?; + try self.asmRegisterRegisterImmediate( + .{ ._ps, .shuf }, + dst_reg.to128(), + dst_reg.to128(), + .u(0), + ); + break :result dst_mcv; + } + }, + 5...8 => if (self.hasFeature(.avx)) { + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_ss, .broadcast }, + dst_reg.to256(), + try src_mcv.mem(self, .{ .size = .dword }), + ) else { + const src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(scalar_ty, src_mcv); + if (self.hasFeature(.avx2)) try self.asmRegisterRegister( + .{ .v_ss, .broadcast }, + dst_reg.to256(), + src_reg.to128(), + ) else { + try self.asmRegisterRegisterRegisterImmediate( + .{ .v_ps, .shuf }, + dst_reg.to128(), + src_reg.to128(), + src_reg.to128(), + .u(0), + ); + try self.asmRegisterRegisterRegisterImmediate( + .{ .v_f128, .insert }, + dst_reg.to256(), + dst_reg.to256(), + dst_reg.to128(), + .u(1), + ); + } + } + break :result .{ .register = dst_reg }; + }, + else => {}, + }, + 64 => switch (vector_len) { + 1 => { + const src_mcv = try self.resolveInst(ty_op.operand); + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); + break :result .{ .register = dst_reg }; + }, + 2 => { + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + if (self.hasFeature(.sse3)) { + if (src_mcv.isBase()) try self.asmRegisterMemory( + if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, + dst_reg.to128(), + try src_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, + dst_reg.to128(), + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(), + ); + break :result .{ .register = dst_reg }; + } else try self.asmRegisterRegister( + .{ ._ps, .movlh }, + dst_reg.to128(), + (if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(), + ); + }, + 3...4 => if (self.hasFeature(.avx)) { + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_sd, .broadcast }, + dst_reg.to256(), + try src_mcv.mem(self, .{ .size = .qword }), + ) else { + const src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(scalar_ty, src_mcv); + if (self.hasFeature(.avx2)) try self.asmRegisterRegister( + .{ .v_sd, .broadcast }, + dst_reg.to256(), + src_reg.to128(), + ) else { + try self.asmRegisterRegister( + .{ .v_, .movddup }, + dst_reg.to128(), + src_reg.to128(), + ); + try self.asmRegisterRegisterRegisterImmediate( + .{ .v_f128, .insert }, + dst_reg.to256(), + dst_reg.to256(), + dst_reg.to128(), + .u(1), + ); + } + } + break :result .{ .register = dst_reg }; + }, + else => {}, + }, + 128 => switch (vector_len) { + 1 => { + const src_mcv = try self.resolveInst(ty_op.operand); + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); + break :result .{ .register = dst_reg }; + }, + 2 => if (self.hasFeature(.avx)) { + const src_mcv = try self.resolveInst(ty_op.operand); + const dst_reg = try self.register_manager.allocReg(inst, dst_rc); + if (src_mcv.isBase()) try self.asmRegisterMemory( + .{ .v_f128, .broadcast }, + dst_reg.to256(), + try src_mcv.mem(self, .{ .size = .xword }), + ) else { + const src_reg = if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(scalar_ty, src_mcv); + try self.asmRegisterRegisterRegisterImmediate( + .{ .v_f128, .insert }, + dst_reg.to256(), + src_reg.to256(), + src_reg.to128(), + .u(1), + ); + } + break :result .{ .register = dst_reg }; + }, + else => {}, + }, + 16, 80 => {}, + else => unreachable, + }, + } + return self.fail("TODO implement airSplat for {f}", .{vector_ty.fmt(pt)}); + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + const ty = self.typeOfIndex(inst); + const vec_len = ty.vectorLen(zcu); + const elem_ty = ty.childType(zcu); + const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const pred_ty = self.typeOf(pl_op.operand); + + const result = result: { + const has_blend = self.hasFeature(.sse4_1); + const has_avx = self.hasFeature(.avx); + const need_xmm0 = has_blend and !has_avx; + const pred_mcv = try self.resolveInst(pl_op.operand); + const mask_reg = mask: { + switch (pred_mcv) { + .register => |pred_reg| switch (pred_reg.class()) { + .general_purpose => {}, + .sse => if (elem_ty.toIntern() == .bool_type) + if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) { + try self.register_manager.getKnownReg(.xmm0, null); + try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{}); + break :mask .xmm0; + } else break :mask if (has_blend) + pred_reg + else + try self.copyToTmpRegister(pred_ty, pred_mcv) + else + return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}), + else => unreachable, + }, + .register_mask => |pred_reg_mask| { + if (pred_reg_mask.info.scalar.bitSize(self.target) != 8 * elem_abi_size) + return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); + + const mask_reg: Register = if (need_xmm0 and pred_reg_mask.reg.id() != comptime Register.xmm0.id()) mask_reg: { + try self.register_manager.getKnownReg(.xmm0, null); + try self.genSetReg(.xmm0, ty, .{ .register = pred_reg_mask.reg }, .{}); + break :mask_reg .xmm0; + } else pred_reg_mask.reg; + const mask_alias = registerAlias(mask_reg, abi_size); + const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); + defer self.register_manager.unlockReg(mask_lock); + + const lhs_mcv = try self.resolveInst(extra.lhs); + const lhs_lock = switch (lhs_mcv) { + .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg), + else => null, + }; + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(extra.rhs); + const rhs_lock = switch (rhs_mcv) { + .register => |rhs_reg| self.register_manager.lockReg(rhs_reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const order = has_blend != pred_reg_mask.info.inverted; + const reuse_mcv, const other_mcv = if (order) + .{ rhs_mcv, lhs_mcv } + else + .{ lhs_mcv, rhs_mcv }; + const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand( + inst, + if (order) extra.rhs else extra.lhs, + @intFromBool(order), + reuse_mcv, + )) reuse_mcv else if (has_avx) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, abi_size); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const mir_tag = @as(?Mir.Inst.FixedTag, if ((pred_reg_mask.info.kind == .all and + elem_ty.toIntern() != .f32_type and elem_ty.toIntern() != .f64_type) or pred_reg_mask.info.scalar == .byte) + if (has_avx) + .{ .vp_b, .blendv } + else if (has_blend) + .{ .p_b, .blendv } + else if (pred_reg_mask.info.kind == .all) + .{ .p_, undefined } + else + null + else if ((pred_reg_mask.info.kind == .all and (elem_ty.toIntern() != .f64_type or !self.hasFeature(.sse2))) or + pred_reg_mask.info.scalar == .dword) + if (has_avx) + .{ .v_ps, .blendv } + else if (has_blend) + .{ ._ps, .blendv } + else if (pred_reg_mask.info.kind == .all) + .{ ._ps, undefined } + else + null + else if (pred_reg_mask.info.kind == .all or pred_reg_mask.info.scalar == .qword) + if (has_avx) + .{ .v_pd, .blendv } + else if (has_blend) + .{ ._pd, .blendv } + else if (pred_reg_mask.info.kind == .all) + .{ ._pd, undefined } + else + null + else + null) orelse return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); + if (has_avx) { + const rhs_alias = if (reuse_mcv.isRegister()) + registerAlias(reuse_mcv.getReg().?, abi_size) + else rhs: { + try self.genSetReg(dst_reg, ty, reuse_mcv, .{}); + break :rhs dst_alias; + }; + if (other_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( + mir_tag, + dst_alias, + rhs_alias, + try other_mcv.mem(self, .{ .size = self.memSize(ty) }), + mask_alias, + ) else try self.asmRegisterRegisterRegisterRegister( + mir_tag, + dst_alias, + rhs_alias, + registerAlias(if (other_mcv.isRegister()) + other_mcv.getReg().? + else + try self.copyToTmpRegister(ty, other_mcv), abi_size), + mask_alias, + ); + } else if (has_blend) if (other_mcv.isBase()) try self.asmRegisterMemoryRegister( + mir_tag, + dst_alias, + try other_mcv.mem(self, .{ .size = self.memSize(ty) }), + mask_alias, + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (other_mcv.isRegister()) + other_mcv.getReg().? + else + try self.copyToTmpRegister(ty, other_mcv), abi_size), + mask_alias, + ) else { + try self.asmRegisterRegister(.{ mir_tag[0], .@"and" }, dst_alias, mask_alias); + if (other_mcv.isBase()) try self.asmRegisterMemory( + .{ mir_tag[0], .andn }, + mask_alias, + try other_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + ) else try self.asmRegisterRegister( + .{ mir_tag[0], .andn }, + mask_alias, + registerAlias(if (other_mcv.isRegister()) + other_mcv.getReg().? + else + try self.copyToTmpRegister(ty, other_mcv), abi_size), + ); + try self.asmRegisterRegister(.{ mir_tag[0], .@"or" }, dst_alias, mask_alias); + } + break :result dst_mcv; + }, + else => {}, + } + const mask_reg: Register = if (need_xmm0) mask_reg: { + try self.register_manager.getKnownReg(.xmm0, null); + break :mask_reg .xmm0; + } else try self.register_manager.allocReg(null, abi.RegisterClass.sse); + const mask_alias = registerAlias(mask_reg, abi_size); + const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); + defer self.register_manager.unlockReg(mask_lock); + + const pred_fits_in_elem = vec_len <= elem_abi_size; + if (self.hasFeature(.avx2) and abi_size <= 32) { + if (pred_mcv.isRegister()) broadcast: { + try self.asmRegisterRegister( + .{ .v_d, .mov }, + mask_reg.to128(), + pred_mcv.getReg().?.to32(), + ); + if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister( + .{ switch (elem_abi_size) { + 1 => .vp_b, + 2 => .vp_w, + 3...4 => .vp_d, + 5...8 => .vp_q, + 9...16 => { + try self.asmRegisterRegisterRegisterImmediate( + .{ .v_f128, .insert }, + mask_alias, + mask_alias, + mask_reg.to128(), + .u(1), + ); + break :broadcast; + }, + 17...32 => break :broadcast, + else => unreachable, + }, .broadcast }, + mask_alias, + mask_reg.to128(), + ); + } else try self.asmRegisterMemory( + .{ switch (vec_len) { + 1...8 => .vp_b, + 9...16 => .vp_w, + 17...32 => .vp_d, + else => unreachable, + }, .broadcast }, + mask_alias, + if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{ + .base = .{ .reg = (try self.copyToTmpRegister( + .usize, + pred_mcv.address(), + )).to64() }, + .mod = .{ .rm = .{ .size = .byte } }, + }, + ); + } else if (abi_size <= 16) broadcast: { + try self.asmRegisterRegister( + .{ if (has_avx) .v_d else ._d, .mov }, + mask_alias, + (if (pred_mcv.isRegister()) + pred_mcv.getReg().? + else + try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(), + ); + if (!pred_fits_in_elem or vec_len == 1) break :broadcast; + if (elem_abi_size <= 1) { + if (has_avx) try self.asmRegisterRegisterRegister( + .{ .vp_, .unpcklbw }, + mask_alias, + mask_alias, + mask_alias, + ) else try self.asmRegisterRegister( + .{ .p_, .unpcklbw }, + mask_alias, + mask_alias, + ); + if (abi_size <= 2) break :broadcast; + } + if (elem_abi_size <= 2) { + try self.asmRegisterRegisterImmediate( + .{ if (has_avx) .vp_w else .p_w, .shufl }, + mask_alias, + mask_alias, + .u(0b00_00_00_00), + ); + if (abi_size <= 8) break :broadcast; + } + try self.asmRegisterRegisterImmediate( + .{ if (has_avx) .vp_d else .p_d, .shuf }, + mask_alias, + mask_alias, + .u(switch (elem_abi_size) { + 1...2, 5...8 => 0b01_00_01_00, + 3...4 => 0b00_00_00_00, + else => unreachable, + }), + ); + } else return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); + const elem_bits: u16 = @intCast(elem_abi_size * 8); + if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) { + const mask_len = elem_abi_size * vec_len; + const mask_ty = try pt.vectorType(.{ + .len = mask_len, + .child = .u8_type, + }); + var mask_elems_buf: [32]u8 = undefined; + const mask_elems = mask_elems_buf[0..mask_len]; + for (mask_elems, 0..) |*elem, bit| elem.* = @intCast(bit / elem_bits); + const mask_mcv = try self.lowerValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = mask_ty.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, mask_elems, .maybe_embedded_nulls) }, + } }))); + const mask_mem: Memory = .{ + .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }; + if (has_avx) try self.asmRegisterRegisterMemory( + .{ .vp_b, .shuf }, + mask_alias, + mask_alias, + mask_mem, + ) else try self.asmRegisterMemory( + .{ .p_b, .shuf }, + mask_alias, + mask_mem, + ); + } else return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); + { + const mask_elem_ty = try pt.intType(.unsigned, elem_bits); + const mask_ty = try pt.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() }); + var mask_elems_buf: [32]InternPool.Index = undefined; + const mask_elems = mask_elems_buf[0..vec_len]; + for (mask_elems, 0..) |*elem, bit| elem.* = (try pt.intValue( + mask_elem_ty, + @as(u8, 1) << @truncate(bit), + )).toIntern(); + const mask_mcv = try self.lowerValue(try pt.aggregateValue(mask_ty, mask_elems)); + const mask_mem: Memory = .{ + .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }; + if (has_avx) { + try self.asmRegisterRegisterMemory( + .{ .vp_, .@"and" }, + mask_alias, + mask_alias, + mask_mem, + ); + try self.asmRegisterRegisterMemory( + .{ switch (elem_abi_size) { + else => unreachable, + 1 => .vp_b, + 2 => .vp_w, + 4 => .vp_d, + 8 => .vp_q, + }, .cmpeq }, + mask_alias, + mask_alias, + mask_mem, + ); + } else { + try self.asmRegisterMemory( + .{ .p_, .@"and" }, + mask_alias, + mask_mem, + ); + try self.asmRegisterMemory( + .{ switch (elem_abi_size) { + else => unreachable, + 1 => .p_b, + 2 => .p_w, + 4 => .p_d, + 8 => if (has_blend) .p_q else unreachable, + }, .cmpeq }, + mask_alias, + mask_mem, + ); + } + } + break :mask mask_reg; + }; + const mask_alias = registerAlias(mask_reg, abi_size); + const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); + defer self.register_manager.unlockReg(mask_lock); + + const lhs_mcv = try self.resolveInst(extra.lhs); + const lhs_lock = switch (lhs_mcv) { + .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg), + else => null, + }; + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(extra.rhs); + const rhs_lock = switch (rhs_mcv) { + .register => |rhs_reg| self.register_manager.lockReg(rhs_reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv; + const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand( + inst, + if (has_blend) extra.rhs else extra.lhs, + @intFromBool(has_blend), + reuse_mcv, + )) reuse_mcv else if (has_avx) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, abi_size); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const mir_tag = @as(?Mir.Inst.FixedTag, switch (elem_ty.zigTypeTag(zcu)) { + else => null, + .int => switch (abi_size) { + 0 => unreachable, + 1...16 => if (has_avx) + .{ .vp_b, .blendv } + else if (has_blend) + .{ .p_b, .blendv } + else + .{ .p_, undefined }, + 17...32 => if (self.hasFeature(.avx2)) + .{ .vp_b, .blendv } + else + null, + else => null, + }, + .float => switch (elem_ty.floatBits(self.target)) { + else => unreachable, + 16, 80, 128 => null, + 32 => switch (vec_len) { + 0 => unreachable, + 1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv }, + 5...8 => if (has_avx) .{ .v_ps, .blendv } else null, + else => null, + }, + 64 => switch (vec_len) { + 0 => unreachable, + 1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv }, + 3...4 => if (has_avx) .{ .v_pd, .blendv } else null, + else => null, + }, + }, + }) orelse return self.fail("TODO implement airSelect for {f}", .{ty.fmt(pt)}); + if (has_avx) { + const rhs_alias = if (rhs_mcv.isRegister()) + registerAlias(rhs_mcv.getReg().?, abi_size) + else rhs: { + try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); + break :rhs dst_alias; + }; + if (lhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( + mir_tag, + dst_alias, + rhs_alias, + try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), + mask_alias, + ) else try self.asmRegisterRegisterRegisterRegister( + mir_tag, + dst_alias, + rhs_alias, + registerAlias(if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + try self.copyToTmpRegister(ty, lhs_mcv), abi_size), + mask_alias, + ); + } else if (has_blend) if (lhs_mcv.isBase()) try self.asmRegisterMemoryRegister( + mir_tag, + dst_alias, + try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), + mask_alias, + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + try self.copyToTmpRegister(ty, lhs_mcv), abi_size), + mask_alias, + ) else { + try self.asmRegisterRegister(.{ mir_tag[0], .@"and" }, dst_alias, mask_alias); + if (rhs_mcv.isBase()) try self.asmRegisterMemory( + .{ mir_tag[0], .andn }, + mask_alias, + try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }), + ) else try self.asmRegisterRegister( + .{ mir_tag[0], .andn }, + mask_alias, + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(ty, rhs_mcv), abi_size), + ); + try self.asmRegisterRegister(.{ mir_tag[0], .@"or" }, dst_alias, mask_alias); + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); +} + +fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data; + + const dst_ty = self.typeOfIndex(inst); + const elem_ty = dst_ty.childType(zcu); + const elem_abi_size: u16 = @intCast(elem_ty.abiSize(zcu)); + const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); + const lhs_ty = self.typeOf(extra.a); + const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); + const rhs_ty = self.typeOf(extra.b); + const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu)); + const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size); + + const ExpectedContents = [32]?i32; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); + const allocator = stack.get(); + + const mask_elems = try allocator.alloc(?i32, extra.mask_len); + defer allocator.free(mask_elems); + for (mask_elems, 0..) |*mask_elem, elem_index| { + const mask_elem_val = + Value.fromInterned(extra.mask).elemValue(pt, elem_index) catch unreachable; + mask_elem.* = if (mask_elem_val.isUndef(zcu)) + null + else + @intCast(mask_elem_val.toSignedInt(zcu)); + } + + const has_avx = self.hasFeature(.avx); + const result = @as(?MCValue, result: { + for (mask_elems) |mask_elem| { + if (mask_elem) |_| break; + } else break :result try self.allocRegOrMem(inst, true); + + for (mask_elems, 0..) |mask_elem, elem_index| { + if (mask_elem orelse continue != elem_index) break; + } else { + const lhs_mcv = try self.resolveInst(extra.a); + if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv; + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{}); + break :result dst_mcv; + } + + for (mask_elems, 0..) |mask_elem, elem_index| { + if (~(mask_elem orelse continue) != elem_index) break; + } else { + const rhs_mcv = try self.resolveInst(extra.b); + if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv; + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{}); + break :result dst_mcv; + } + + for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { + if (elem_abi_size > 8) break :unpck; + if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck; + + var sources: [2]?u1 = @splat(null); + for (mask_elems, 0..) |maybe_mask_elem, elem_index| { + const mask_elem = maybe_mask_elem orelse continue; + const mask_elem_index = + std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; + const elem_byte = (elem_index >> 1) * elem_abi_size; + if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) { + .unpckl => 0b0000, + .unpckh => 0b1000, + else => unreachable, + }) | (elem_byte << 1 & 0b10000)) break :unpck; + + const source = @intFromBool(mask_elem < 0); + if (sources[elem_index & 0b00001]) |prev_source| { + if (source != prev_source) break :unpck; + } else sources[elem_index & 0b00001] = source; + } + if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck; + + const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; + const operand_tys = [2]Type{ lhs_ty, rhs_ty }; + const lhs_mcv = try self.resolveInst(operands[sources[0].?]); + const rhs_mcv = try self.resolveInst(operands[sources[1].?]); + + const dst_mcv: MCValue = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) + lhs_mcv + else if (has_avx and lhs_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, max_abi_size); + + const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or + (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) { + 4 => if (has_avx) .v_ps else ._ps, + 8 => if (has_avx) .v_pd else ._pd, + else => unreachable, + }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) { + .unpckl => switch (elem_abi_size) { + 1 => .unpcklbw, + 2 => .unpcklwd, + 4 => .unpckldq, + 8 => .unpcklqdq, + else => unreachable, + }, + .unpckh => switch (elem_abi_size) { + 1 => .unpckhbw, + 2 => .unpckhwd, + 4 => .unpckhdq, + 8 => .unpckhqdq, + else => unreachable, + }, + else => unreachable, + } }; + if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory( + mir_tag, + dst_alias, + registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_alias, + registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), + ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory( + mir_tag, + dst_alias, + try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + ) else try self.asmRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), + ); + break :result dst_mcv; + } + + pshufd: { + if (elem_abi_size != 4) break :pshufd; + if (max_abi_size > self.vectorSize(.float)) break :pshufd; + + var control: u8 = 0b00_00_00_00; + var sources: [1]?u1 = @splat(null); + for (mask_elems, 0..) |maybe_mask_elem, elem_index| { + const mask_elem = maybe_mask_elem orelse continue; + const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); + if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd; + + const source = @intFromBool(mask_elem < 0); + if (sources[0]) |prev_source| { + if (source != prev_source) break :pshufd; + } else sources[(elem_index & 0b010) >> 1] = source; + + const select_bit: u3 = @intCast((elem_index & 0b011) << 1); + const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; + if (elem_index & 0b100 == 0) + control |= select_mask + else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd; + } + + const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; + const operand_tys = [2]Type{ lhs_ty, rhs_ty }; + const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]); + + const dst_reg = if (src_mcv.isRegister() and + self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv)) + src_mcv.getReg().? + else + try self.register_manager.allocReg(inst, abi.RegisterClass.sse); + const dst_alias = registerAlias(dst_reg, max_abi_size); + + if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( + .{ if (has_avx) .vp_d else .p_d, .shuf }, + dst_alias, + try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + .u(control), + ) else try self.asmRegisterRegisterImmediate( + .{ if (has_avx) .vp_d else .p_d, .shuf }, + dst_alias, + registerAlias(if (src_mcv.isRegister()) + src_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size), + .u(control), + ); + break :result .{ .register = dst_reg }; + } + + shufps: { + if (elem_abi_size != 4) break :shufps; + if (max_abi_size > self.vectorSize(.float)) break :shufps; + + var control: u8 = 0b00_00_00_00; + var sources: [2]?u1 = @splat(null); + for (mask_elems, 0..) |maybe_mask_elem, elem_index| { + const mask_elem = maybe_mask_elem orelse continue; + const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); + if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps; + + const source = @intFromBool(mask_elem < 0); + if (sources[(elem_index & 0b010) >> 1]) |prev_source| { + if (source != prev_source) break :shufps; + } else sources[(elem_index & 0b010) >> 1] = source; + + const select_bit: u3 = @intCast((elem_index & 0b011) << 1); + const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; + if (elem_index & 0b100 == 0) + control |= select_mask + else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps; + } + if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps; + + const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; + const operand_tys = [2]Type{ lhs_ty, rhs_ty }; + const lhs_mcv = try self.resolveInst(operands[sources[0].?]); + const rhs_mcv = try self.resolveInst(operands[sources[1].?]); + + const dst_mcv: MCValue = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) + lhs_mcv + else if (has_avx and lhs_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, max_abi_size); + + if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .v_ps, .shuf }, + dst_alias, + registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + .u(control), + ) else try self.asmRegisterRegisterRegisterImmediate( + .{ .v_ps, .shuf }, + dst_alias, + registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), + .u(control), + ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( + .{ ._ps, .shuf }, + dst_alias, + try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + .u(control), + ) else try self.asmRegisterRegisterImmediate( + .{ ._ps, .shuf }, + dst_alias, + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), + .u(control), + ); + break :result dst_mcv; + } + + shufpd: { + if (elem_abi_size != 8) break :shufpd; + if (max_abi_size > self.vectorSize(.float)) break :shufpd; + + var control: u4 = 0b0_0_0_0; + var sources: [2]?u1 = @splat(null); + for (mask_elems, 0..) |maybe_mask_elem, elem_index| { + const mask_elem = maybe_mask_elem orelse continue; + const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); + if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd; + + const source = @intFromBool(mask_elem < 0); + if (sources[elem_index & 0b01]) |prev_source| { + if (source != prev_source) break :shufpd; + } else sources[elem_index & 0b01] = source; + + control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index); + } + if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd; + + const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b }; + const operand_tys: [2]Type = .{ lhs_ty, rhs_ty }; + const lhs_mcv = try self.resolveInst(operands[sources[0].?]); + const rhs_mcv = try self.resolveInst(operands[sources[1].?]); + + const dst_mcv: MCValue = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) + lhs_mcv + else if (has_avx and lhs_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, max_abi_size); + + if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .v_pd, .shuf }, + dst_alias, + registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + .u(control), + ) else try self.asmRegisterRegisterRegisterImmediate( + .{ .v_pd, .shuf }, + dst_alias, + registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), + .u(control), + ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( + .{ ._pd, .shuf }, + dst_alias, + try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), + .u(control), + ) else try self.asmRegisterRegisterImmediate( + .{ ._pd, .shuf }, + dst_alias, + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), + .u(control), + ); + break :result dst_mcv; + } + + blend: { + if (elem_abi_size < 2) break :blend; + if (dst_abi_size > self.vectorSize(.float)) break :blend; + if (!self.hasFeature(.sse4_1)) break :blend; + + var control: u8 = 0b0_0_0_0_0_0_0_0; + for (mask_elems, 0..) |maybe_mask_elem, elem_index| { + const mask_elem = maybe_mask_elem orelse continue; + const mask_elem_index = + std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; + if (mask_elem_index != elem_index) break :blend; + + const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); + if (elem_index & 0b1000 == 0) + control |= select_mask + else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend; + } + + if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: { + const expanded_control = switch (elem_abi_size) { + 4 => control, + 8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | + @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | + @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | + @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00), + else => break :vpblendd, + }; + + const lhs_mcv = try self.resolveInst(extra.a); + const lhs_reg = if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, lhs_mcv); + const lhs_lock = self.register_manager.lockReg(lhs_reg); + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_mcv = try self.resolveInst(extra.b); + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); + if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_d, .blend }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(lhs_reg, dst_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + .u(expanded_control), + ) else try self.asmRegisterRegisterRegisterImmediate( + .{ .vp_d, .blend }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(lhs_reg, dst_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + .u(expanded_control), + ); + break :result .{ .register = dst_reg }; + } + + if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: { + const expanded_control = switch (elem_abi_size) { + 2 => control, + 4 => if (dst_abi_size <= 16 or + @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0))) + @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | + @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | + @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | + @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00) + else + break :pblendw, + 8 => if (dst_abi_size <= 16 or + @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0))) + @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) | + @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000) + else + break :pblendw, + 16 => break :pblendw, + else => unreachable, + }; + + const lhs_mcv = try self.resolveInst(extra.a); + const rhs_mcv = try self.resolveInst(extra.b); + + const dst_mcv: MCValue = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, extra.a, 0, lhs_mcv)) + lhs_mcv + else if (has_avx and lhs_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); + const dst_reg = dst_mcv.getReg().?; + + if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + .{ .vp_w, .blend }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + dst_reg, dst_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + .u(expanded_control), + ) else try self.asmRegisterRegisterRegisterImmediate( + .{ .vp_w, .blend }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + dst_reg, dst_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + .u(expanded_control), + ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( + .{ .p_w, .blend }, + registerAlias(dst_reg, dst_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + .u(expanded_control), + ) else try self.asmRegisterRegisterImmediate( + .{ .p_w, .blend }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + .u(expanded_control), + ); + break :result .{ .register = dst_reg }; + } + + const expanded_control = switch (elem_abi_size) { + 4, 8 => control, + 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) | + @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00), + else => unreachable, + }; + + const lhs_mcv = try self.resolveInst(extra.a); + const rhs_mcv = try self.resolveInst(extra.b); + + const dst_mcv: MCValue = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, extra.a, 0, lhs_mcv)) + lhs_mcv + else if (has_avx and lhs_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); + const dst_reg = dst_mcv.getReg().?; + + if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( + switch (elem_abi_size) { + 4 => .{ .v_ps, .blend }, + 8, 16 => .{ .v_pd, .blend }, + else => unreachable, + }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + dst_reg, dst_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + .u(expanded_control), + ) else try self.asmRegisterRegisterRegisterImmediate( + switch (elem_abi_size) { + 4 => .{ .v_ps, .blend }, + 8, 16 => .{ .v_pd, .blend }, + else => unreachable, + }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + dst_reg, dst_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + .u(expanded_control), + ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( + switch (elem_abi_size) { + 4 => .{ ._ps, .blend }, + 8, 16 => .{ ._pd, .blend }, + else => unreachable, + }, + registerAlias(dst_reg, dst_abi_size), + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + .u(expanded_control), + ) else try self.asmRegisterRegisterImmediate( + switch (elem_abi_size) { + 4 => .{ ._ps, .blend }, + 8, 16 => .{ ._pd, .blend }, + else => unreachable, + }, + registerAlias(dst_reg, dst_abi_size), + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + .u(expanded_control), + ); + break :result .{ .register = dst_reg }; + } + + blendv: { + if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv; + + const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); + const select_mask_ty = try pt.vectorType(.{ + .len = @intCast(mask_elems.len), + .child = select_mask_elem_ty.toIntern(), + }); + var select_mask_elems: [32]InternPool.Index = undefined; + for ( + select_mask_elems[0..mask_elems.len], + mask_elems, + 0.., + ) |*select_mask_elem, maybe_mask_elem, elem_index| { + const mask_elem = maybe_mask_elem orelse continue; + const mask_elem_index = + std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; + if (mask_elem_index != elem_index) break :blendv; + + select_mask_elem.* = (if (mask_elem < 0) + try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty) + else + try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern(); + } + const select_mask_mcv = try self.lowerValue( + try pt.aggregateValue(select_mask_ty, select_mask_elems[0..mask_elems.len]), + ); + + if (self.hasFeature(.sse4_1)) { + const mir_tag: Mir.Inst.FixedTag = .{ + if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or + (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) { + 4 => if (has_avx) .v_ps else ._ps, + 8 => if (has_avx) .v_pd else ._pd, + else => unreachable, + } else if (has_avx) .vp_b else .p_b, + .blendv, + }; + + const select_mask_reg = if (!has_avx) reg: { + try self.register_manager.getKnownReg(.xmm0, null); + try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{}); + break :reg .xmm0; + } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); + const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size); + const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg); + defer self.register_manager.unlockReg(select_mask_lock); + + const lhs_mcv = try self.resolveInst(extra.a); + const rhs_mcv = try self.resolveInst(extra.b); + + const dst_mcv: MCValue = if (lhs_mcv.isRegister() and + self.reuseOperand(inst, extra.a, 0, lhs_mcv)) + lhs_mcv + else if (has_avx and lhs_mcv.isRegister()) + .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, dst_abi_size); + + if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( + mir_tag, + dst_alias, + if (lhs_mcv.isRegister()) + registerAlias(lhs_mcv.getReg().?, dst_abi_size) + else + dst_alias, + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + select_mask_alias, + ) else try self.asmRegisterRegisterRegisterRegister( + mir_tag, + dst_alias, + if (lhs_mcv.isRegister()) + registerAlias(lhs_mcv.getReg().?, dst_abi_size) + else + dst_alias, + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + select_mask_alias, + ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister( + mir_tag, + dst_alias, + try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + select_mask_alias, + ) else try self.asmRegisterRegisterRegister( + mir_tag, + dst_alias, + registerAlias(if (rhs_mcv.isRegister()) + rhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), + select_mask_alias, + ); + break :result dst_mcv; + } + + const lhs_mcv = try self.resolveInst(extra.a); + const rhs_mcv = try self.resolveInst(extra.b); + + const dst_mcv: MCValue = if (rhs_mcv.isRegister() and + self.reuseOperand(inst, extra.b, 1, rhs_mcv)) + rhs_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv); + const dst_reg = dst_mcv.getReg().?; + const dst_alias = registerAlias(dst_reg, dst_abi_size); + + const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); + const mask_alias = registerAlias(mask_reg, dst_abi_size); + const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); + defer self.register_manager.unlockReg(mask_lock); + + const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat()) + switch (elem_ty.floatBits(self.target)) { + 16, 80, 128 => .p_, + 32 => ._ps, + 64 => ._pd, + else => unreachable, + } + else + .p_; + try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); + if (lhs_mcv.isBase()) try self.asmRegisterMemory( + .{ mir_fixes, .andn }, + mask_alias, + try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), + ) else try self.asmRegisterRegister( + .{ mir_fixes, .andn }, + mask_alias, + if (lhs_mcv.isRegister()) + lhs_mcv.getReg().? + else + try self.copyToTmpRegister(dst_ty, lhs_mcv), + ); + try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); + break :result dst_mcv; + } + + pshufb: { + if (max_abi_size > 16) break :pshufb; + if (!self.hasFeature(.ssse3)) break :pshufb; + + const temp_regs = + try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse); + const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs); + defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); + + const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size); + try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{}); + + const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size); + try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{}); + + var lhs_mask_elems: [16]InternPool.Index = undefined; + for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| { + const elem_index = byte_index / elem_abi_size; + lhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { + const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; + if (mask_elem < 0) break :elem 0b1_00_00000; + const mask_elem_index: u31 = @intCast(mask_elem); + const byte_off: u32 = @intCast(byte_index % elem_abi_size); + break :elem mask_elem_index * elem_abi_size + byte_off; + })).toIntern(); + } + const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); + const lhs_mask_mcv = try self.lowerValue( + try pt.aggregateValue(lhs_mask_ty, lhs_mask_elems[0..max_abi_size]), + ); + const lhs_mask_mem: Memory = .{ + .base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) }, + .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, + }; + if (has_avx) try self.asmRegisterRegisterMemory( + .{ .vp_b, .shuf }, + lhs_temp_alias, + lhs_temp_alias, + lhs_mask_mem, + ) else try self.asmRegisterMemory( + .{ .p_b, .shuf }, + lhs_temp_alias, + lhs_mask_mem, + ); + + var rhs_mask_elems: [16]InternPool.Index = undefined; + for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| { + const elem_index = byte_index / elem_abi_size; + rhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { + const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; + if (mask_elem >= 0) break :elem 0b1_00_00000; + const mask_elem_index: u31 = @intCast(~mask_elem); + const byte_off: u32 = @intCast(byte_index % elem_abi_size); + break :elem mask_elem_index * elem_abi_size + byte_off; + })).toIntern(); + } + const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); + const rhs_mask_mcv = try self.lowerValue( + try pt.aggregateValue(rhs_mask_ty, rhs_mask_elems[0..max_abi_size]), + ); + const rhs_mask_mem: Memory = .{ + .base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) }, + .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, + }; + if (has_avx) try self.asmRegisterRegisterMemory( + .{ .vp_b, .shuf }, + rhs_temp_alias, + rhs_temp_alias, + rhs_mask_mem, + ) else try self.asmRegisterMemory( + .{ .p_b, .shuf }, + rhs_temp_alias, + rhs_mask_mem, + ); + + if (has_avx) try self.asmRegisterRegisterRegister( + .{ switch (elem_ty.zigTypeTag(zcu)) { + else => break :result null, + .int => .vp_, + .float => switch (elem_ty.floatBits(self.target)) { + 32 => .v_ps, + 64 => .v_pd, + 16, 80, 128 => break :result null, + else => unreachable, + }, + }, .@"or" }, + lhs_temp_alias, + lhs_temp_alias, + rhs_temp_alias, + ) else try self.asmRegisterRegister( + .{ switch (elem_ty.zigTypeTag(zcu)) { + else => break :result null, + .int => .p_, + .float => switch (elem_ty.floatBits(self.target)) { + 32 => ._ps, + 64 => ._pd, + 16, 80, 128 => break :result null, + else => unreachable, + }, + }, .@"or" }, + lhs_temp_alias, + rhs_temp_alias, + ); + break :result .{ .register = temp_regs[0] }; + } + + break :result null; + }) orelse return self.fail("TODO implement airShuffle from {f} and {f} to {f} with {f}", .{ + lhs_ty.fmt(pt), + rhs_ty.fmt(pt), + dst_ty.fmt(pt), + Value.fromInterned(extra.mask).fmtValue(pt), + }); + return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); +} + +fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const result_ty = self.typeOfIndex(inst); + const len: usize = @intCast(result_ty.arrayLen(zcu)); + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); + const result: MCValue = result: { + switch (result_ty.zigTypeTag(zcu)) { + .@"struct" => { + const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu)); + if (result_ty.containerLayout(zcu) == .@"packed") { + const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern()); + try self.genInlineMemset( + .{ .lea_frame = .{ .index = frame_index } }, + .{ .immediate = 0 }, + .{ .immediate = result_ty.abiSize(zcu) }, + .{}, + ); + for (elements, 0..) |elem, elem_i_usize| { + const elem_i: u32 = @intCast(elem_i_usize); + if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; + + const elem_ty = result_ty.fieldType(elem_i, zcu); + const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu)); + if (elem_bit_size > 64) { + return self.fail( + "TODO airAggregateInit implement packed structs with large fields", + .{}, + ); + } + const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); + const elem_abi_bits = elem_abi_size * 8; + const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i); + const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size); + const elem_bit_off = elem_off % elem_abi_bits; + const elem_mcv = try self.resolveInst(elem); + const elem_lock = switch (elem_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + .immediate => |imm| lock: { + if (imm == 0) continue; + break :lock null; + }, + else => null, + }; + defer if (elem_lock) |lock| self.register_manager.unlockReg(lock); + + const elem_extra_bits = self.regExtraBits(elem_ty); + { + const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); + const temp_alias = registerAlias(temp_reg, elem_abi_size); + const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); + defer self.register_manager.unlockReg(temp_lock); + + if (elem_bit_off < elem_extra_bits) { + try self.truncateRegister(elem_ty, temp_alias); + } + if (elem_bit_off > 0) try self.genShiftBinOpMir( + .{ ._l, .sh }, + elem_ty, + .{ .register = temp_alias }, + .u8, + .{ .immediate = elem_bit_off }, + ); + try self.genBinOpMir( + .{ ._, .@"or" }, + elem_ty, + .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } }, + .{ .register = temp_alias }, + ); + } + if (elem_bit_off > elem_extra_bits) { + const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); + const temp_alias = registerAlias(temp_reg, elem_abi_size); + const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); + defer self.register_manager.unlockReg(temp_lock); + + if (elem_extra_bits > 0) { + try self.truncateRegister(elem_ty, temp_alias); + } + try self.genShiftBinOpMir( + .{ ._r, .sh }, + elem_ty, + .{ .register = temp_reg }, + .u8, + .{ .immediate = elem_abi_bits - elem_bit_off }, + ); + try self.genBinOpMir( + .{ ._, .@"or" }, + elem_ty, + .{ .load_frame = .{ + .index = frame_index, + .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)), + } }, + .{ .register = temp_alias }, + ); + } + } + } else for (elements, 0..) |elem, elem_i| { + if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; + + const elem_ty = result_ty.fieldType(elem_i, zcu); + const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu)); + const elem_mcv = try self.resolveInst(elem); + try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv, .{}); + } + break :result .{ .load_frame = .{ .index = frame_index } }; + }, + .array, .vector => { + const elem_ty = result_ty.childType(zcu); + if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) { + const result_size: u32 = @intCast(result_ty.abiSize(zcu)); + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); + try self.asmRegisterRegister( + .{ ._, .xor }, + registerAlias(dst_reg, @min(result_size, 4)), + registerAlias(dst_reg, @min(result_size, 4)), + ); + + for (elements, 0..) |elem, elem_i| { + const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem }); + const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); + defer self.register_manager.unlockReg(elem_lock); + + try self.asmRegisterImmediate( + .{ ._, .@"and" }, + registerAlias(elem_reg, @min(result_size, 4)), + .u(1), + ); + if (elem_i > 0) try self.asmRegisterImmediate( + .{ ._l, .sh }, + registerAlias(elem_reg, result_size), + .u(@intCast(elem_i)), + ); + try self.asmRegisterRegister( + .{ ._, .@"or" }, + registerAlias(dst_reg, result_size), + registerAlias(elem_reg, result_size), + ); + } + break :result .{ .register = dst_reg }; + } else { + const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu)); + const elem_size: u32 = @intCast(elem_ty.abiSize(zcu)); + + for (elements, 0..) |elem, elem_i| { + const elem_mcv = try self.resolveInst(elem); + const elem_off: i32 = @intCast(elem_size * elem_i); + try self.genSetMem( + .{ .frame = frame_index }, + elem_off, + elem_ty, + elem_mcv, + .{}, + ); + } + if (result_ty.sentinel(zcu)) |sentinel| try self.genSetMem( + .{ .frame = frame_index }, + @intCast(elem_size * elements.len), + elem_ty, + try self.lowerValue(sentinel), + .{}, + ); + break :result .{ .load_frame = .{ .index = frame_index } }; + } + }, + else => unreachable, + } + }; + + if (elements.len <= Air.Liveness.bpi - 1) { + var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); + @memcpy(buf[0..elements.len], elements); + return self.finishAir(inst, result, buf); + } + var bt = self.liveness.iterateBigTomb(inst); + for (elements) |elem| try self.feed(&bt, elem); + return self.finishAirResult(inst, result); +} + +fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; + const result: MCValue = result: { + const union_ty = self.typeOfIndex(inst); + const layout = union_ty.unionGetLayout(zcu); + + const src_ty = self.typeOf(extra.init); + const src_mcv = try self.resolveInst(extra.init); + if (layout.tag_size == 0) { + if (layout.abi_size <= src_ty.abiSize(zcu) and + self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv; + + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); + break :result dst_mcv; + } + + const dst_mcv = try self.allocRegOrMem(inst, false); + + const loaded_union = zcu.typeToUnion(union_ty).?; + const field_name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index]; + const tag_ty: Type = .fromInterned(loaded_union.enum_tag_ty); + const field_index = tag_ty.enumFieldIndex(field_name, zcu).?; + const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index); + const tag_int_val = try tag_val.intFromEnum(tag_ty, pt); + const tag_int = tag_int_val.toUnsignedInt(zcu); + const tag_off: i32 = @intCast(layout.tagOffset()); + try self.genCopy( + tag_ty, + dst_mcv.address().offset(tag_off).deref(), + .{ .immediate = tag_int }, + .{}, + ); + + const pl_off: i32 = @intCast(layout.payloadOffset()); + try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{}); + + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ extra.init, .none, .none }); +} + +fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + const ty = self.typeOfIndex(inst); + + const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand }; + const result = result: { + if (switch (ty.scalarType(zcu).floatBits(self.target)) { + 16, 80, 128 => true, + 32, 64 => !self.hasFeature(.fma), + else => unreachable, + }) { + if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {f}", .{ + ty.fmt(pt), + }); + + var sym_buf: ["__fma?".len]u8 = undefined; + break :result try self.genCall(.{ .extern_func = .{ + .return_type = ty.toIntern(), + .param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() }, + .sym = std.fmt.bufPrint(&sym_buf, "{s}fma{s}", .{ + floatLibcAbiPrefix(ty), + floatLibcAbiSuffix(ty), + }) catch unreachable, + } }, &.{ ty, ty, ty }, &.{ + .{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand }, + }, .{}); + } + + var mcvs: [3]MCValue = undefined; + var locks: [3]?RegisterManager.RegisterLock = @splat(null); + defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); + var order: [3]u2 = @splat(0); + var unused: std.StaticBitSet(3) = .initFull(); + for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| { + const op_index: u2 = @intCast(op_i); + mcv.* = try self.resolveInst(op); + if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) { + order[op_index] = 1; + unused.unset(0); + } else if (unused.isSet(2) and mcv.isBase()) { + order[op_index] = 3; + unused.unset(2); + } + switch (mcv.*) { + .register => |reg| lock.* = self.register_manager.lockReg(reg), + else => {}, + } + } + for (&order, &mcvs, &locks) |*mop_index, *mcv, *lock| { + if (mop_index.* != 0) continue; + mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?)); + if (mop_index.* > 1 and mcv.isRegister()) continue; + const reg = try self.copyToTmpRegister(ty, mcv.*); + mcv.* = .{ .register = reg }; + if (lock.*) |old_lock| self.register_manager.unlockReg(old_lock); + lock.* = self.register_manager.lockRegAssumeUnused(reg); + } + + const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or + std.mem.eql(u2, &order, &.{ 3, 1, 2 })) + switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(self.target)) { + 32 => .{ .v_ss, .fmadd132 }, + 64 => .{ .v_sd, .fmadd132 }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (ty.childType(zcu).floatBits(self.target)) { + 32 => switch (ty.vectorLen(zcu)) { + 1 => .{ .v_ss, .fmadd132 }, + 2...8 => .{ .v_ps, .fmadd132 }, + else => null, + }, + 64 => switch (ty.vectorLen(zcu)) { + 1 => .{ .v_sd, .fmadd132 }, + 2...4 => .{ .v_pd, .fmadd132 }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + } + else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 })) + switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(self.target)) { + 32 => .{ .v_ss, .fmadd213 }, + 64 => .{ .v_sd, .fmadd213 }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (ty.childType(zcu).floatBits(self.target)) { + 32 => switch (ty.vectorLen(zcu)) { + 1 => .{ .v_ss, .fmadd213 }, + 2...8 => .{ .v_ps, .fmadd213 }, + else => null, + }, + 64 => switch (ty.vectorLen(zcu)) { + 1 => .{ .v_sd, .fmadd213 }, + 2...4 => .{ .v_pd, .fmadd213 }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + } + else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 })) + switch (ty.zigTypeTag(zcu)) { + .float => switch (ty.floatBits(self.target)) { + 32 => .{ .v_ss, .fmadd231 }, + 64 => .{ .v_sd, .fmadd231 }, + 16, 80, 128 => null, + else => unreachable, + }, + .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { + .float => switch (ty.childType(zcu).floatBits(self.target)) { + 32 => switch (ty.vectorLen(zcu)) { + 1 => .{ .v_ss, .fmadd231 }, + 2...8 => .{ .v_ps, .fmadd231 }, + else => null, + }, + 64 => switch (ty.vectorLen(zcu)) { + 1 => .{ .v_sd, .fmadd231 }, + 2...4 => .{ .v_pd, .fmadd231 }, + else => null, + }, + 16, 80, 128 => null, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + } + else + unreachable) orelse return self.fail("TODO implement airMulAdd for {f}", .{ty.fmt(pt)}); + + var mops: [3]MCValue = undefined; + for (order, mcvs) |mop_index, mcv| mops[mop_index - 1] = mcv; + + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const mop1_reg = registerAlias(mops[0].getReg().?, abi_size); + const mop2_reg = registerAlias(mops[1].getReg().?, abi_size); + if (mops[2].isRegister()) try self.asmRegisterRegisterRegister( + mir_tag, + mop1_reg, + mop2_reg, + registerAlias(mops[2].getReg().?, abi_size), + ) else try self.asmRegisterRegisterMemory( + mir_tag, + mop1_reg, + mop2_reg, + try mops[2].mem(self, .{ .size = .fromSize(abi_size) }), + ); + break :result mops[0]; + }; + return self.finishAir(inst, result, ops); +} + +fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty; + const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque); + + const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) { + .x86_64_sysv => result: { + const info = self.va_info.sysv; + const dst_fi = try self.allocFrameIndex(.initSpill(va_list_ty, zcu)); + var field_off: u31 = 0; + // gp_offset: c_uint, + try self.genSetMem( + .{ .frame = dst_fi }, + field_off, + .c_uint, + .{ .immediate = info.gp_count * 8 }, + .{}, + ); + field_off += @intCast(Type.c_uint.abiSize(zcu)); + // fp_offset: c_uint, + try self.genSetMem( + .{ .frame = dst_fi }, + field_off, + .c_uint, + .{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 }, + .{}, + ); + field_off += @intCast(Type.c_uint.abiSize(zcu)); + // overflow_arg_area: *anyopaque, + try self.genSetMem( + .{ .frame = dst_fi }, + field_off, + ptr_anyopaque_ty, + .{ .lea_frame = info.overflow_arg_area }, + .{}, + ); + field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu)); + // reg_save_area: *anyopaque, + try self.genSetMem( + .{ .frame = dst_fi }, + field_off, + ptr_anyopaque_ty, + .{ .lea_frame = info.reg_save_area }, + .{}, + ); + field_off += @intCast(ptr_anyopaque_ty.abiSize(zcu)); + break :result .{ .load_frame = .{ .index = dst_fi } }; + }, + .x86_64_win => return self.fail("TODO implement c_va_start for Win64", .{}), + else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}), + }; + return self.finishAir(inst, result, .{ .none, .none, .none }); +} + +fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const ty = self.typeOfIndex(inst); + const promote_ty = self.promoteVarArg(ty); + const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque); + const unused = self.liveness.isUnused(inst); + + const result: MCValue = switch (self.fn_type.fnCallingConvention(zcu)) { + .x86_64_sysv => result: { + try self.spillEflagsIfOccupied(); + + const tmp_regs = + try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp); + const offset_reg = tmp_regs[0].to32(); + const addr_reg = tmp_regs[1].to64(); + const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); + defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); + + const promote_mcv = try self.allocTempRegOrMem(promote_ty, true); + const promote_lock = switch (promote_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (promote_lock) |lock| self.register_manager.unlockReg(lock); + + const ptr_arg_list_reg = + try self.copyToTmpRegister(self.typeOf(ty_op.operand), .{ .air_ref = ty_op.operand }); + const ptr_arg_list_lock = self.register_manager.lockRegAssumeUnused(ptr_arg_list_reg); + defer self.register_manager.unlockReg(ptr_arg_list_lock); + + const gp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 0 } }; + const fp_offset: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 4 } }; + const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } }; + const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } }; + + const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target, .arg), .none); + switch (classes[0]) { + .integer => { + assert(classes.len == 1); + + try self.genSetReg(offset_reg, .c_uint, gp_offset, .{}); + try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u( + abi.SysV.c_abi_int_param_regs.len * 8, + )); + const mem_reloc = try self.asmJccReloc(.ae, undefined); + + try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{}); + if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{ + .base = .{ .reg = addr_reg }, + .mod = .{ .rm = .{ .index = offset_reg.to64() } }, + }); + try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{ + .base = .{ .reg = offset_reg.to64() }, + .mod = .{ .rm = .{ .disp = 8 } }, + }); + try self.genCopy(.c_uint, gp_offset, .{ .register = offset_reg }, .{}); + const done_reloc = try self.asmJmpReloc(undefined); + + self.performReloc(mem_reloc); + try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{}); + try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{ + .base = .{ .reg = addr_reg }, + .mod = .{ .rm = .{ .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)) } }, + }); + try self.genCopy( + ptr_anyopaque_ty, + overflow_arg_area, + .{ .register = offset_reg.to64() }, + .{}, + ); + + self.performReloc(done_reloc); + if (!unused) try self.genCopy(promote_ty, promote_mcv, .{ + .indirect = .{ .reg = addr_reg }, + }, .{}); + }, + .sse => { + assert(classes.len == 1); + + try self.genSetReg(offset_reg, .c_uint, fp_offset, .{}); + try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u( + abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16, + )); + const mem_reloc = try self.asmJccReloc(.ae, undefined); + + try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{}); + if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{ + .base = .{ .reg = addr_reg }, + .mod = .{ .rm = .{ .index = offset_reg.to64() } }, + }); + try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{ + .base = .{ .reg = offset_reg.to64() }, + .mod = .{ .rm = .{ .disp = 16 } }, + }); + try self.genCopy(.c_uint, fp_offset, .{ .register = offset_reg }, .{}); + const done_reloc = try self.asmJmpReloc(undefined); + + self.performReloc(mem_reloc); + try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{}); + try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{ + .base = .{ .reg = addr_reg }, + .mod = .{ .rm = .{ .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)) } }, + }); + try self.genCopy( + ptr_anyopaque_ty, + overflow_arg_area, + .{ .register = offset_reg.to64() }, + .{}, + ); + + self.performReloc(done_reloc); + if (!unused) try self.genCopy(promote_ty, promote_mcv, .{ + .indirect = .{ .reg = addr_reg }, + }, .{}); + }, + .memory => { + assert(classes.len == 1); + unreachable; + }, + else => return self.fail("TODO implement c_va_arg for {f} on SysV", .{promote_ty.fmt(pt)}), + } + + if (unused) break :result .unreach; + if (ty.toIntern() == promote_ty.toIntern()) break :result promote_mcv; + + if (!promote_ty.isRuntimeFloat()) { + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(ty, dst_mcv, promote_mcv, .{}); + break :result dst_mcv; + } + + assert(ty.toIntern() == .f32_type and promote_ty.toIntern() == .f64_type); + const dst_mcv = if (promote_mcv.isRegister()) + promote_mcv + else + try self.copyToRegisterWithInstTracking(inst, ty, promote_mcv); + const dst_reg = dst_mcv.getReg().?.to128(); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + if (self.hasFeature(.avx)) if (promote_mcv.isBase()) try self.asmRegisterRegisterMemory( + .{ .v_ss, .cvtsd2 }, + dst_reg, + dst_reg, + try promote_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegisterRegister( + .{ .v_ss, .cvtsd2 }, + dst_reg, + dst_reg, + (if (promote_mcv.isRegister()) + promote_mcv.getReg().? + else + try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), + ) else if (promote_mcv.isBase()) try self.asmRegisterMemory( + .{ ._ss, .cvtsd2 }, + dst_reg, + try promote_mcv.mem(self, .{ .size = .qword }), + ) else try self.asmRegisterRegister( + .{ ._ss, .cvtsd2 }, + dst_reg, + (if (promote_mcv.isRegister()) + promote_mcv.getReg().? + else + try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), + ); + break :result promote_mcv; + }, + .x86_64_win => return self.fail("TODO implement c_va_arg for Win64", .{}), + else => |cc| return self.fail("{s} does not support var args", .{@tagName(cc)}), + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + +fn airVaCopy(self: *CodeGen, inst: Air.Inst.Index) !void { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const ptr_va_list_ty = self.typeOf(ty_op.operand); + + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.load(dst_mcv, ptr_va_list_ty, .{ .air_ref = ty_op.operand }); + return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); +} + +fn airVaEnd(self: *CodeGen, inst: Air.Inst.Index) !void { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + return self.finishAir(inst, .unreach, .{ un_op, .none, .none }); +} + +fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue { + const zcu = self.pt.zcu; + const ty = self.typeOf(ref); + + // If the type has no codegen bits, no need to store it. + if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none; + + const mcv: MCValue = if (ref.toIndex()) |inst| mcv: { + break :mcv self.inst_tracking.getPtr(inst).?.short; + } else mcv: { + break :mcv try self.lowerValue(.fromInterned(ref.toInterned().?)); + }; + + switch (mcv) { + .none, .unreach, .dead => unreachable, + else => return mcv, + } +} + +fn getResolvedInstValue(self: *CodeGen, inst: Air.Inst.Index) *InstTracking { + const tracking = self.inst_tracking.getPtr(inst).?; + return switch (tracking.short) { + .none, .unreach, .dead => unreachable, + else => tracking, + }; +} + +/// If the MCValue is an immediate, and it does not fit within this type, +/// we put it in a register. +/// A potential opportunity for future optimization here would be keeping track +/// of the fact that the instruction is available both as an immediate +/// and as a register. +fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue { + const mcv = try self.resolveInst(operand); + const ti = @typeInfo(T).int; + switch (mcv) { + .immediate => |imm| { + // This immediate is unsigned. + const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed)); + if (imm >= std.math.maxInt(U)) { + return MCValue{ .register = try self.copyToTmpRegister(.usize, mcv) }; + } + }, + else => {}, + } + return mcv; +} + +fn lowerValue(cg: *CodeGen, val: Value) Allocator.Error!MCValue { + return switch (try codegen.lowerValue(cg.pt, val, cg.target)) { + .none => .none, + .undef => .undef, + .immediate => |imm| .{ .immediate = imm }, + .lea_nav => |nav| .{ .lea_nav = nav }, + .lea_uav => |uav| .{ .lea_uav = uav }, + .load_uav => |uav| .{ .load_uav = uav }, + }; +} + +const CallMCValues = struct { + args: []MCValue, + air_arg_count: u32, + return_value: InstTracking, + stack_byte_count: u31, + stack_align: InternPool.Alignment, + gp_count: u32, + fp_count: u32, + err_ret_trace_reg: Register, + + fn deinit(self: *CallMCValues, func: *CodeGen) void { + func.gpa.free(self.args); + self.* = undefined; + } +}; + +const win64_shadow_space = 8 * 4; + +/// Caller must call `CallMCValues.deinit`. +fn resolveCallingConventionValues( + cg: *CodeGen, + fn_info: InternPool.Key.FuncType, + var_args: []const Type, + stack_frame_base: FrameIndex, +) !CallMCValues { + const pt = cg.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const cc = fn_info.cc; + + const ExpectedContents = extern struct { + param_types: [32][@sizeOf(Type)]u8 align(@alignOf(Type)), + }; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); + const allocator = stack.get(); + + const param_types = try allocator.alloc(Type, fn_info.param_types.len + var_args.len); + defer allocator.free(param_types); + + for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*param_ty, arg_ty| + param_ty.* = .fromInterned(arg_ty); + for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty| + param_ty.* = cg.promoteVarArg(arg_ty); + + var result: CallMCValues = .{ + .args = try cg.gpa.alloc(MCValue, param_types.len), + .air_arg_count = 0, + // These undefined values must be populated before returning from this function. + .return_value = undefined, + .stack_byte_count = 0, + .stack_align = undefined, + .gp_count = 0, + .fp_count = 0, + .err_ret_trace_reg = .none, + }; + errdefer cg.gpa.free(result.args); + + const ret_ty: Type = .fromInterned(fn_info.return_type); + switch (cc) { + .naked => { + assert(result.args.len == 0); + result.return_value = .init(.unreach); + result.stack_align = switch (cg.target.cpu.arch) { + else => unreachable, + .x86 => .@"4", + .x86_64 => .@"8", + }; + }, + .x86_64_sysv, .x86_64_win => |cc_opts| { + const param_gpr = abi.getCAbiIntParamRegs(cc); + var param_gpr_index: u32 = 0; + const param_sse = abi.getCAbiSseParamRegs(cc, cg.target); + var param_sse_index: u32 = 0; + + result.stack_align = .fromByteUnits(cc_opts.incoming_stack_alignment orelse 16); + + switch (cc) { + .x86_64_sysv => {}, + .x86_64_win => result.stack_byte_count += @intCast(win64_shadow_space), + else => unreachable, + } + + // Return values + if (ret_ty.isNoReturn(zcu)) { + result.return_value = .init(.unreach); + } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + // TODO: is this even possible for C calling convention? + result.return_value = .init(.none); + } else { + var ret_tracking: [4]InstTracking = undefined; + var ret_tracking_len: u32 = 0; + var ret_gpr = abi.getCAbiIntReturnRegs(cc); + var ret_sse = abi.getCAbiSseReturnRegs(cc); + var ret_x87 = abi.getCAbiX87ReturnRegs(cc); + + const classes = switch (cc) { + .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, cg.target, .ret), .none), + .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu, cg.target)}, + else => unreachable, + }; + for (classes) |class| switch (class) { + .integer => { + ret_tracking[ret_tracking_len] = .init(.{ .register = registerAlias( + ret_gpr[0], + @intCast(@min(ret_ty.abiSize(zcu), 8)), + ) }); + ret_tracking_len += 1; + ret_gpr = ret_gpr[1..]; + }, + .sse, .float, .float_combine, .win_i128 => { + const abi_size: u32 = @intCast(ret_ty.abiSize(zcu)); + const reg_size = @min(abi_size, cg.vectorSize(.float)); + var byte_offset: u32 = 0; + while (byte_offset < abi_size) : (byte_offset += reg_size) { + const ret_sse_reg = registerAlias(ret_sse[0], reg_size); + ret_sse = ret_sse[1..]; + + ret_tracking[ret_tracking_len] = .init(.{ .register = ret_sse_reg }); + ret_tracking_len += 1; + } + }, + .sseup => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.sse)), + .x87 => { + ret_tracking[ret_tracking_len] = .init(.{ .register = ret_x87[0] }); + ret_tracking_len += 1; + ret_x87 = ret_x87[1..]; + }, + .x87up => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.x87)), + .memory, .integer_per_element => { + ret_tracking[ret_tracking_len] = .{ + .short = .{ .indirect = .{ .reg = ret_gpr[0].to64() } }, + .long = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }, + }; + ret_tracking_len += 1; + ret_gpr = ret_gpr[1..]; + param_gpr_index += 1; + }, + .none => unreachable, + }; + result.return_value = switch (ret_tracking_len) { + else => unreachable, + 1 => ret_tracking[0], + 2 => .init(.{ .register_pair = .{ + ret_tracking[0].short.register, + ret_tracking[1].short.register, + } }), + 3 => .init(.{ .register_triple = .{ + ret_tracking[0].short.register, + ret_tracking[1].short.register, + ret_tracking[2].short.register, + } }), + 4 => .init(.{ .register_quadruple = .{ + ret_tracking[0].short.register, + ret_tracking[1].short.register, + ret_tracking[2].short.register, + ret_tracking[3].short.register, + } }), + }; + } + + // Input params + params: for (param_types, result.args) |ty, *arg| { + assert(ty.hasRuntimeBitsIgnoreComptime(zcu)); + result.air_arg_count += 1; + switch (cc) { + .x86_64_sysv => {}, + .x86_64_win => { + param_gpr_index = @max(param_gpr_index, param_sse_index); + param_sse_index = param_gpr_index; + }, + else => unreachable, + } + + var arg_mcv: [4]MCValue = undefined; + var arg_mcv_len: u32 = 0; + + const classes = switch (cc) { + .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .arg), .none), + .x86_64_win => &.{abi.classifyWindows(ty, zcu, cg.target)}, + else => unreachable, + }; + classes: for (classes) |class| switch (class) { + .integer => { + if (param_gpr_index >= param_gpr.len) break; + arg_mcv[arg_mcv_len] = .{ + .register = registerAlias(param_gpr[param_gpr_index], @intCast(@min(ty.abiSize(zcu), 8))), + }; + arg_mcv_len += 1; + param_gpr_index += 1; + }, + .sse, .float, .float_combine => { + const abi_size: u32 = @intCast(ty.abiSize(zcu)); + const reg_size = @min(abi_size, cg.vectorSize(.float)); + var byte_offset: u32 = 0; + while (byte_offset < abi_size) : (byte_offset += reg_size) { + if (param_sse_index >= param_sse.len) break :classes; + + const param_sse_reg = registerAlias(param_sse[param_sse_index], reg_size); + param_sse_index += 1; + + arg_mcv[arg_mcv_len] = .{ .register = param_sse_reg }; + arg_mcv_len += 1; + } + }, + .sseup => assert(arg_mcv[arg_mcv_len - 1].register.isClass(.sse)), + .x87, .x87up, .memory, .win_i128 => switch (cc) { + .x86_64_sysv => switch (class) { + .x87, .x87up, .memory => break, + else => unreachable, + }, + .x86_64_win => if (ty.abiSize(zcu) > 8) { + if (param_gpr_index < param_gpr.len) { + arg_mcv[arg_mcv_len] = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }; + arg_mcv_len += 1; + param_gpr_index += 1; + } else { + assert(arg_mcv_len == 0); + const param_align = Type.usize.abiAlignment(zcu); + result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); + result.stack_align = result.stack_align.max(param_align); + arg.* = .{ .indirect_load_frame = .{ + .index = stack_frame_base, + .off = result.stack_byte_count, + } }; + result.stack_byte_count += @intCast(Type.usize.abiSize(zcu)); + continue :params; + } + } else break, + else => unreachable, + }, + .none => unreachable, + .integer_per_element => { + const remaining_param_gpr_len: u3 = @intCast(param_gpr.len - param_gpr_index); + param_gpr_index = @intCast(param_gpr.len); + + const frame_elem_align = 8; + const frame_elems_len = ty.vectorLen(zcu) - remaining_param_gpr_len; + const frame_elem_size = std.mem.alignForward(u64, ty.childType(zcu).abiSize(zcu), frame_elem_align); + const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size); + + result.stack_byte_count = std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align); + arg_mcv[arg_mcv_len] = .{ .elementwise_args = .{ + .regs = remaining_param_gpr_len, + .frame_off = @intCast(result.stack_byte_count), + .frame_index = stack_frame_base, + } }; + arg_mcv_len += 1; + result.stack_byte_count += frame_size; + }, + } else { + arg.* = switch (arg_mcv_len) { + else => unreachable, + 1 => arg_mcv[0], + 2 => .{ .register_pair = .{ + arg_mcv[0].register, + arg_mcv[1].register, + } }, + 3 => .{ .register_triple = .{ + arg_mcv[0].register, + arg_mcv[1].register, + arg_mcv[2].register, + } }, + 4 => .{ .register_quadruple = .{ + arg_mcv[0].register, + arg_mcv[1].register, + arg_mcv[2].register, + arg_mcv[3].register, + } }, + }; + continue; + } + + const param_align = ty.abiAlignment(zcu).max(.@"8"); + result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); + result.stack_align = result.stack_align.max(param_align); + arg.* = .{ .load_frame = .{ + .index = stack_frame_base, + .off = result.stack_byte_count, + } }; + result.stack_byte_count += @intCast(ty.abiSize(zcu)); + } + assert(param_gpr_index <= 6); + result.gp_count = param_gpr_index; + assert(param_sse_index <= 16); + result.fp_count = param_sse_index; + }, + .auto => { + result.stack_align = abi.zigcc.stack_align orelse .fromByteUnits(cg.vectorSize(.float)); + + var param_gpr = abi.getCAbiIntParamRegs(cc); + var param_x87 = abi.getCAbiX87ParamRegs(cc); + var param_sse = abi.getCAbiSseParamRegs(cc, cg.target); + + if (zcu.comp.config.any_error_tracing) { + result.err_ret_trace_reg = param_gpr[param_gpr.len - 1]; + param_gpr = param_gpr[0 .. param_gpr.len - 1]; + } + + // Return values + result.return_value = if (ret_ty.isNoReturn(zcu)) + .init(.unreach) + else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) + .init(.none) + else return_value: { + const ret_gpr = abi.getCAbiIntReturnRegs(cc); + const ret_size: u31 = @intCast(ret_ty.abiSize(zcu)); + if (abi.zigcc.return_in_regs) switch (cg.regClassForType(ret_ty)) { + .general_purpose, .gphi => if (ret_size <= @as(u4, switch (cg.target.cpu.arch) { + else => unreachable, + .x86 => 4, + .x86_64 => 8, + })) + break :return_value .init(.{ .register = registerAlias(ret_gpr[0], ret_size) }) + else if (ret_gpr.len >= 2 and ret_ty.isSliceAtRuntime(zcu)) + break :return_value .init(.{ .register_pair = ret_gpr[0..2].* }), + .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87 => if (ret_size <= 16) break :return_value .init(.{ .register = .st0 }), + .sse => if (ret_size <= cg.vectorSize(.float)) break :return_value .init(.{ + .register = registerAlias(abi.getCAbiSseReturnRegs(cc)[0], @max(ret_size, 16)), + }), + }; + const ret_indirect_reg = param_gpr[0]; + param_gpr = param_gpr[1..]; + break :return_value .{ + .short = .{ .indirect = .{ .reg = ret_gpr[0] } }, + .long = .{ .indirect = .{ .reg = ret_indirect_reg } }, + }; + }; + + // Input params + for (param_types, result.args) |param_ty, *arg| { + if (!param_ty.hasRuntimeBitsIgnoreComptime(zcu)) { + arg.* = .none; + continue; + } + result.air_arg_count += 1; + const param_size: u31 = @intCast(param_ty.abiSize(zcu)); + if (abi.zigcc.params_in_regs) switch (cg.regClassForType(param_ty)) { + .general_purpose, .gphi => if (param_gpr.len >= 1 and param_size <= @as(u4, switch (cg.target.cpu.arch) { + else => unreachable, + .x86 => 4, + .x86_64 => 8, + })) { + arg.* = .{ .register = registerAlias(param_gpr[0], param_size) }; + param_gpr = param_gpr[1..]; + continue; + } else if (param_gpr.len >= 2 and param_ty.isSliceAtRuntime(zcu)) { + arg.* = .{ .register_pair = param_gpr[0..2].* }; + param_gpr = param_gpr[2..]; + continue; + }, + .segment, .mmx, .ip, .cr, .dr => unreachable, + .x87 => if (param_x87.len >= 1 and param_size <= 16) { + arg.* = .{ .register = param_x87[0] }; + param_x87 = param_x87[1..]; + continue; + }, + .sse => if (param_sse.len >= 1 and param_size <= cg.vectorSize(.float)) { + arg.* = .{ + .register = registerAlias(param_sse[0], @max(param_size, 16)), + }; + param_sse = param_sse[1..]; + continue; + }, + }; + const param_align = param_ty.abiAlignment(zcu); + result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); + result.stack_align = result.stack_align.max(param_align); + arg.* = .{ .load_frame = .{ + .index = stack_frame_base, + .off = result.stack_byte_count, + } }; + result.stack_byte_count += param_size; + } + }, + else => return cg.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}), + } + + result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count)); + return result; +} + +fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = cg.pt.zcu; + return switch (cg.owner) { + .nav_index => |i| zcu.codegenFail(i, format, args), + .lazy_sym => |s| zcu.codegenFailType(s.ty, format, args), + }; +} + +fn failMsg(cg: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } { + @branchHint(.cold); + const zcu = cg.pt.zcu; + return switch (cg.owner) { + .nav_index => |i| zcu.codegenFailMsg(i, msg), + .lazy_sym => |s| zcu.codegenFailTypeMsg(s.ty, msg), + }; +} + +fn parseRegName(name: []const u8) ?Register { + if (std.mem.startsWith(u8, name, "db")) return @enumFromInt( + @intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null), + ); + return std.meta.stringToEnum(Register, name); +} + +/// Returns register wide enough to hold at least `size_bytes`. +fn registerAlias(reg: Register, size_bytes: u32) Register { + if (size_bytes == 0) unreachable; // should be comptime-known + return switch (reg.class()) { + .general_purpose => if (size_bytes <= 1) + reg.to8() + else if (size_bytes <= 2) + reg.to16() + else if (size_bytes <= 4) + reg.to32() + else if (size_bytes <= 8) + reg.to64() + else + unreachable, + .gphi => if (size_bytes <= 1) + reg + else if (size_bytes <= 2) + reg.to16() + else if (size_bytes <= 4) + reg.to32() + else if (size_bytes <= 8) + reg.to64() + else + unreachable, + .segment => if (size_bytes <= 2) + reg + else + unreachable, + .x87 => if (size_bytes >= 4 and size_bytes <= 16) + reg + else + unreachable, + .mmx => if (size_bytes <= 8) + reg + else + unreachable, + .sse => if (size_bytes <= 16) + reg.to128() + else if (size_bytes <= 32) + reg.to256() + else + unreachable, + .ip => if (size_bytes <= 2) + .ip + else if (size_bytes <= 4) + .eip + else if (size_bytes <= 8) + .rip + else + unreachable, + .cr => if (size_bytes <= 8) + reg + else + unreachable, + .dr => if (size_bytes <= 8) + reg + else + unreachable, + }; +} + +fn memSize(self: *CodeGen, ty: Type) Memory.Size { + const zcu = self.pt.zcu; + return if (self.floatBits(ty)) |float_bits| + .fromBitSize(float_bits) + else if (ty.isVector(zcu) and ty.vectorLen(zcu) == 1 and self.floatBits(ty.childType(zcu)) == 80) + .tbyte + else + .fromSize(@intCast(ty.abiSize(zcu))); +} + +fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type { + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + var parts: [parts_len]Type = undefined; + switch (ip.indexToKey(ty.toIntern())) { + .vector_type => |vector_type| if (std.math.divExact(u32, vector_type.len, parts_len)) |vec_len| return .{ + try pt.vectorType(.{ .len = vec_len, .child = vector_type.child }), + } ** parts_len else |err| switch (err) { + error.DivisionByZero => unreachable, + error.UnexpectedRemainder => {}, + }, + .tuple_type => |tuple_type| if (tuple_type.types.len == parts_len) { + for (&parts, tuple_type.types.get(ip)) |*part, part_ty| part.* = .fromInterned(part_ty); + return parts; + }, + else => {}, + } + const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .other), .none); + if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| { + part.* = switch (class) { + .integer => if (part_i < parts_len - 1) + .u64 + else part: { + const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; + const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8)); + break :part switch (@divExact(ty.abiSize(zcu) - part_i * 8, elem_size)) { + 1 => elem_ty, + else => |array_len| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }), + }; + }, + .float => .f32, + .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }), + .sse => .f64, + else => break, + }; + } else { + var part_sizes: u64 = 0; + for (parts) |part| part_sizes += part.abiSize(zcu); + if (part_sizes == ty.abiSize(zcu)) return parts; + }; + return self.fail("TODO implement splitType({d}, {f})", .{ parts_len, ty.fmt(pt) }); +} + +/// Truncates the value in the register in place. +/// Clobbers any remaining bits. +fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void { + const pt = self.pt; + const zcu = pt.zcu; + const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{ + .signedness = .unsigned, + .bits = @intCast(ty.bitSize(zcu)), + }; + const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return; + try self.spillEflagsIfOccupied(); + switch (int_info.signedness) { + .signed => { + try self.genShiftBinOpMir(.{ ._l, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift }); + try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift }); + }, + .unsigned => { + const mask = ~@as(u64, 0) >> shift; + if (int_info.bits <= 32) { + try self.genBinOpMir(.{ ._, .@"and" }, .u32, .{ .register = reg }, .{ .immediate = mask }); + } else { + const tmp_reg = try self.copyToTmpRegister(.usize, .{ .immediate = mask }); + try self.genBinOpMir(.{ ._, .@"and" }, .usize, .{ .register = reg }, .{ .register = tmp_reg }); + } + }, + } +} + +fn regBitSize(self: *CodeGen, ty: Type) u64 { + const zcu = self.pt.zcu; + const abi_size = ty.abiSize(zcu); + return switch (ty.zigTypeTag(zcu)) { + else => switch (abi_size) { + 1 => 8, + 2 => 16, + 3...4 => 32, + 5...8 => 64, + else => unreachable, + }, + .float => switch (abi_size) { + 1...16 => 128, + 17...32 => 256, + else => unreachable, + }, + }; +} + +fn regExtraBits(self: *CodeGen, ty: Type) u64 { + return self.regBitSize(ty) - ty.bitSize(self.pt.zcu); +} + +fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool { + return switch (feature) { + .@"64bit" => switch (cg.target.cpu.arch) { + else => unreachable, + .x86 => false, + .x86_64 => true, + }, + .false_deps_getmant, + .false_deps_lzcnt_tzcnt, + .false_deps_mulc, + .false_deps_mullq, + .false_deps_perm, + .false_deps_popcnt, + .false_deps_range, + .slow_3ops_lea, + .slow_incdec, + .slow_lea, + .slow_pmaddwd, + .slow_pmulld, + .slow_shld, + .slow_two_mem_ops, + .slow_unaligned_mem_16, + .slow_unaligned_mem_32, + => switch (cg.mod.optimize_mode) { + .Debug, .ReleaseSafe, .ReleaseFast => null, + .ReleaseSmall => false, + }, + .fast_11bytenop, + .fast_15bytenop, + .fast_7bytenop, + .fast_bextr, + .fast_dpwssd, + .fast_gather, + .fast_hops, + .fast_imm16, + .fast_lzcnt, + .fast_movbe, + .fast_scalar_fsqrt, + .fast_scalar_shift_masks, + .fast_shld_rotate, + .fast_variable_crosslane_shuffle, + .fast_variable_perlane_shuffle, + .fast_vector_fsqrt, + .fast_vector_shift_masks, + => switch (cg.mod.optimize_mode) { + .Debug, .ReleaseSafe, .ReleaseFast => null, + .ReleaseSmall => true, + }, + .mmx => false, + .sahf => switch (cg.target.cpu.arch) { + else => unreachable, + .x86 => true, + .x86_64 => null, + }, + else => null, + } orelse cg.target.cpu.has(.x86, feature); +} + +fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type { + const pt = self.pt; + const zcu = pt.zcu; + return self.air.typeOf(inst, &zcu.intern_pool); +} + +fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type { + return Temp.typeOf(.{ .index = inst }, self); +} + +fn intCompilerRtAbiName(int_bits: u32) u8 { + return switch (int_bits) { + 1...32 => 's', + 33...64 => 'd', + 65...128 => 't', + else => unreachable, + }; +} + +fn floatCompilerRtAbiName(float_bits: u32) u8 { + return switch (float_bits) { + 16 => 'h', + 32 => 's', + 64 => 'd', + 80 => 'x', + 128 => 't', + else => unreachable, + }; +} + +fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type { + if (ty.toIntern() == .f16_type and + (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and + self.target.os.tag.isDarwin()) return .u16; + return ty; +} + +fn floatLibcAbiPrefix(ty: Type) []const u8 { + return switch (ty.toIntern()) { + .f16_type, .f80_type => "__", + .f32_type, .f64_type, .f128_type, .c_longdouble_type => "", + else => unreachable, + }; +} + +fn floatLibcAbiSuffix(ty: Type) []const u8 { + return switch (ty.toIntern()) { + .f16_type => "h", + .f32_type => "f", + .f64_type => "", + .f80_type => "x", + .f128_type => "q", + .c_longdouble_type => "l", + else => unreachable, + }; +} + +fn promoteInt(self: *CodeGen, ty: Type) Type { + const pt = self.pt; + const zcu = pt.zcu; + const int_info: InternPool.Key.IntType = switch (ty.toIntern()) { + .bool_type => .{ .signedness = .unsigned, .bits = 1 }, + else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty, + }; + for ([_]Type{ + .c_int, .c_uint, + .c_long, .c_ulong, + .c_longlong, .c_ulonglong, + }) |promote_ty| { + const promote_info = promote_ty.intInfo(zcu); + if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue; + if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and + promote_info.signedness == .signed) <= promote_info.bits) return promote_ty; + } + return ty; +} + +fn promoteVarArg(self: *CodeGen, ty: Type) Type { + if (!ty.isRuntimeFloat()) return self.promoteInt(ty); + switch (ty.floatBits(self.target)) { + 32, 64 => return .f64, + else => |float_bits| { + assert(float_bits == self.target.cTypeBitSize(.longdouble)); + return .c_longdouble; + }, + } +} + +fn unalignedSize(cg: *CodeGen, ty: Type) u64 { + const zcu = cg.pt.zcu; + return switch (zcu.intern_pool.indexToKey(ty.toIntern())) { + .vector_type => |vector_type| switch (vector_type.child) { + .bool_type => ty.abiSize(zcu), + else => Type.fromInterned(vector_type.child).abiSize(zcu) * vector_type.len, + }, + else => ty.abiSize(zcu), + }; +} + +fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 { + const zcu = cg.pt.zcu; + return switch (zcu.intern_pool.indexToKey(ty.toIntern())) { + .vector_type => |vector_type| switch (vector_type.child) { + .bool_type => vector_type.len, + else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)), + }, + else => @intCast(ty.bitSize(zcu)), + }; +} + +fn intInfo(cg: *CodeGen, ty: Type) ?std.builtin.Type.Int { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + var ty_index = ty.ip_index; + while (true) switch (ip.indexToKey(ty_index)) { + .int_type => |int_type| return int_type, + .ptr_type => |ptr_type| return switch (ptr_type.flags.size) { + .one, .many, .c => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() }, + .slice => null, + }, + .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBitsIgnoreComptime(zcu)) + .{ .signedness = .unsigned, .bits = 1 } + else switch (ip.indexToKey(opt_child)) { + .ptr_type => |ptr_type| switch (ptr_type.flags.size) { + .one, .many => switch (ptr_type.flags.is_allowzero) { + false => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() }, + true => null, + }, + .slice, .c => null, + }, + else => null, + }, + .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type) + .hasRuntimeBitsIgnoreComptime(zcu)) .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } else null, + .simple_type => |simple_type| return switch (simple_type) { + .bool => .{ .signedness = .unsigned, .bits = 1 }, + .anyerror => .{ .signedness = .unsigned, .bits = zcu.errorSetBits() }, + .isize => .{ .signedness = .signed, .bits = cg.target.ptrBitWidth() }, + .usize => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() }, + .c_char => .{ .signedness = cg.target.cCharSignedness(), .bits = cg.target.cTypeBitSize(.char) }, + .c_short => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.short) }, + .c_ushort => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.short) }, + .c_int => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.int) }, + .c_uint => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.int) }, + .c_long => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.long) }, + .c_ulong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.long) }, + .c_longlong => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.longlong) }, + .c_ulonglong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.longlong) }, + .f16, .f32, .f64, .f80, .f128, .c_longdouble => null, + .anyopaque, + .void, + .type, + .comptime_int, + .comptime_float, + .noreturn, + .null, + .undefined, + .enum_literal, + .adhoc_inferred_error_set, + .generic_poison, + => unreachable, + }, + .struct_type => { + const loaded_struct = ip.loadStructType(ty_index); + switch (loaded_struct.layout) { + .auto, .@"extern" => return null, + .@"packed" => ty_index = loaded_struct.backingIntTypeUnordered(ip), + } + }, + .union_type => return switch (ip.loadUnionType(ty_index).flagsUnordered(ip).layout) { + .auto, .@"extern" => null, + .@"packed" => .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) }, + }, + .enum_type => ty_index = ip.loadEnumType(ty_index).tag_ty, + .error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = zcu.errorSetBits() }, + else => return null, + }; +} + +fn floatBits(cg: *CodeGen, ty: Type) ?u16 { + return if (ty.isRuntimeFloat()) ty.floatBits(cg.target) else null; +} + +const Temp = struct { + index: Air.Inst.Index, + + fn unwrap(temp: Temp, cg: *CodeGen) union(enum) { + ref: Air.Inst.Ref, + temp: Index, + err_ret_trace, + } { + switch (temp.index.unwrap()) { + .ref => |ref| return .{ .ref = ref }, + .target => |target_index| { + if (temp.index == err_ret_trace_index) return .err_ret_trace; + const temp_index: Index = @enumFromInt(target_index); + assert(temp_index.isValid(cg)); + return .{ .temp = temp_index }; + }, + } + } + + fn typeOf(temp: Temp, cg: *CodeGen) Type { + return switch (temp.unwrap(cg)) { + .ref => switch (cg.air.instructions.items(.tag)[@intFromEnum(temp.index)]) { + .loop_switch_br => cg.typeOf(cg.air.unwrapSwitch(temp.index).operand), + else => cg.air.typeOfIndex(temp.index, &cg.pt.zcu.intern_pool), + }, + .temp => |temp_index| temp_index.typeOf(cg), + .err_ret_trace => .usize, + }; + } + + fn isMut(temp: Temp, cg: *CodeGen) bool { + return switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => false, + .temp => |temp_index| switch (temp_index.tracking(cg).short) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .eflags, + .register_offset, + .register_mask, + .memory, + .indirect, + .indirect_load_frame, + .lea_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .lea_extern_func, + .load_extern_func, + .elementwise_args, + .reserved_frame, + .air_ref, + => false, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + => true, + .load_frame => |frame_addr| !frame_addr.index.isNamed(), + }, + }; + } + + fn tracking(temp: Temp, cg: *CodeGen) InstTracking { + return cg.inst_tracking.get(temp.index).?; + } + + fn getOffset(temp: Temp, off: i32, cg: *CodeGen) InnerError!Temp { + const new_temp_index = cg.next_temp_index; + cg.temp_type[@intFromEnum(new_temp_index)] = .usize; + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + const mcv = temp.tracking(cg).short; + switch (mcv) { + else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), + .register => |reg| { + const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ + .base = .{ .reg = reg.to64() }, + .mod = .{ .rm = .{ .disp = off } }, + }); + }, + .register_offset => |reg_off| { + const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ + .base = .{ .reg = reg_off.reg.to64() }, + .mod = .{ .rm = .{ .disp = reg_off.off + off } }, + }); + }, + .load_frame, .load_nav, .lea_nav, .load_uav, .lea_uav, .load_lazy_sym, .lea_lazy_sym => { + const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{ .reg = new_reg, .off = off } }); + try cg.genSetReg(new_reg, .usize, mcv, .{}); + }, + .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }), + } + return .{ .index = new_temp_index.toIndex() }; + } + + fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) InnerError!void { + if (off == 0) return; + switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => {}, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + switch (temp_tracking.short) { + else => {}, + .register => |reg| { + try cg.freeValue(temp_tracking.long, .{}); + temp_tracking.* = .init(.{ .register_offset = .{ + .reg = reg, + .off = off, + } }); + return; + }, + .register_offset => |reg_off| { + try cg.freeValue(temp_tracking.long, .{}); + temp_tracking.* = .init(.{ .register_offset = .{ + .reg = reg_off.reg, + .off = reg_off.off + off, + } }); + return; + }, + .lea_frame => |frame_addr| { + assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr)); + temp_tracking.* = .init(.{ .lea_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }); + return; + }, + } + }, + } + const new_temp = try temp.getOffset(off, cg); + try temp.die(cg); + temp.* = new_temp; + } + + fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) InnerError!Temp { + const new_temp_index = cg.next_temp_index; + cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty; + switch (temp.tracking(cg).short) { + else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), + .immediate => |imm| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .immediate = imm }); + }, + .register => |reg| { + assert(limb_index == 0); + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64()); + }, + .register_pair => |regs| { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64()); + }, + .register_offset => |reg_off| { + assert(limb_index == 0); + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ + .base = .{ .reg = reg_off.reg.to64() }, + .mod = .{ .rm = .{ .disp = reg_off.off + @as(u31, limb_index) * 8 } }, + }); + }, + .load_frame => |frame_addr| { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = frame_addr.off + @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_frame => |frame_addr| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr }); + }, + .load_nav => |nav| { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .nav = nav }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_nav => |nav| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_nav = nav }); + }, + .load_uav => |uav| { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .uav = uav }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_uav => |uav| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_uav = uav }); + }, + .load_lazy_sym => |lazy_sym| { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .lazy_sym = lazy_sym }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_lazy_sym => |lazy_sym| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_lazy_sym = lazy_sym }); + }, + .load_extern_func => |extern_func| { + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .extern_func = extern_func }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_extern_func => |extern_func| { + assert(limb_index == 0); + new_temp_index.tracking(cg).* = .init(.{ .lea_extern_func = extern_func }); + }, + } + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + return .{ .index = new_temp_index.toIndex() }; + } + + fn getByteLen(temp: *Temp, cg: *CodeGen) Select.Error!Temp { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + const ptr_info = ip.indexToKey(temp.typeOf(cg).toIntern()).ptr_type; + switch (ptr_info.flags.size) { + .one => { + const array_info = ip.indexToKey(ptr_info.child).array_type; + return cg.tempInit(.usize, .{ + .immediate = Type.fromInterned(array_info.child).abiSize(zcu) * array_info.len, + }); + }, + .many, .c => unreachable, + .slice => { + const elem_size = Type.fromInterned(ptr_info.child).abiSize(zcu); + var len = try temp.getLimb(.usize, 1, cg); + while (try len.toRegClass(true, .general_purpose, cg)) {} + const len_reg = len.tracking(cg).short.register.to64(); + if (!std.math.isPowerOfTwo(elem_size)) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + len_reg, + len_reg, + .u(elem_size), + ); + } else if (elem_size > 8) { + try cg.spillEflagsIfOccupied(); + try cg.asmRegisterImmediate( + .{ ._l, .sh }, + len_reg, + .u(std.math.log2_int(u64, elem_size)), + ); + } else if (elem_size != 1) try cg.asmRegisterMemory(.{ ._, .lea }, len_reg, .{ + .mod = .{ .rm = .{ + .index = len_reg, + .scale = .fromFactor(@intCast(elem_size)), + } }, + }); + return len; + }, + } + } + + fn toLimb(temp: *Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) InnerError!void { + switch (temp.unwrap(cg)) { + .ref => {}, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + switch (temp_tracking.short) { + else => {}, + .register, .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => { + assert(limb_index == 0); + cg.temp_type[@intFromEnum(temp_index)] = limb_ty; + return; + }, + .register_pair => |regs| { + switch (temp_tracking.long) { + .none, .reserved_frame => {}, + else => temp_tracking.long = + temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(), + } + for (regs, 0..) |reg, reg_index| if (reg_index != limb_index) + cg.register_manager.freeReg(reg); + temp_tracking.* = .init(.{ .register = regs[limb_index] }); + cg.temp_type[@intFromEnum(temp_index)] = limb_ty; + return; + }, + .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) { + assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr)); + temp_tracking.* = .init(.{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + @as(u31, limb_index) * 8, + } }); + cg.temp_type[@intFromEnum(temp_index)] = limb_ty; + return; + }, + } + }, + .err_ret_trace => unreachable, + } + const new_temp = try temp.getLimb(limb_ty, limb_index, cg); + try temp.die(cg); + temp.* = new_temp; + } + + fn toSlicePtr(temp: *Temp, cg: *CodeGen) InnerError!void { + const temp_ty = temp.typeOf(cg); + if (temp_ty.isSlice(cg.pt.zcu)) try temp.toLimb(temp_ty.slicePtrFieldType(cg.pt.zcu), 0, cg); + } + + fn toSliceLen(temp: *Temp, cg: *CodeGen) InnerError!void { + try temp.toLimb(.usize, 1, cg); + } + + fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) InnerError!bool { + const ty: Type = ty: switch (temp.unwrap(cg)) { + .ref => |ref| cg.typeOf(ref), + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + if (temp_tracking.short == .register and + temp_tracking.short.register == new_reg) return false; + break :ty temp_index.typeOf(cg); + }, + .err_ret_trace => .usize, + }; + const new_temp_index = cg.next_temp_index; + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + try cg.register_manager.getReg(new_reg, new_temp_index.toIndex()); + cg.temp_type[@intFromEnum(new_temp_index)] = ty; + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + while (try temp.toBase(false, cg)) {} + try temp.readTo(ty, .{ .register = new_reg }, .{}, cg); + try temp.die(cg); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + fn toRegPair(temp: *Temp, new_regs: [2]Register, cg: *CodeGen) InnerError!bool { + const ty: Type = ty: switch (temp.unwrap(cg)) { + .ref => |ref| cg.typeOf(ref), + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + if (temp_tracking.short == .register_pair and + std.mem.eql(Register, &temp_tracking.short.register_pair, &new_regs)) return false; + break :ty temp_index.typeOf(cg); + }, + .err_ret_trace => .usize, + }; + const new_temp_index = cg.next_temp_index; + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + for (new_regs) |new_reg| try cg.register_manager.getReg(new_reg, new_temp_index.toIndex()); + cg.temp_type[@intFromEnum(new_temp_index)] = ty; + new_temp_index.tracking(cg).* = .init(.{ .register_pair = new_regs }); + while (try temp.toBase(false, cg)) {} + for (new_regs, 0..) |new_reg, reg_index| try temp.readTo( + .usize, + .{ .register = new_reg }, + .{ .disp = @intCast(8 * reg_index) }, + cg, + ); + try temp.die(cg); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + fn toRegClass(temp: *Temp, mut: bool, rc: Register.Class, cg: *CodeGen) InnerError!bool { + const val = temp.tracking(cg).short; + if (!mut or temp.isMut(cg)) switch (val) { + else => {}, + .register => |reg| if (reg.isClass(rc)) return false, + .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.isClass(rc)) return false, + }; + const ty = temp.typeOf(cg); + const new_temp_index = cg.next_temp_index; + cg.temp_type[@intFromEnum(new_temp_index)] = ty; + const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc)); + try cg.genSetReg(new_reg, ty, val, .{}); + new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); + try temp.die(cg); + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) InnerError!void { + while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| { + if (try part_temp.toRegClass(true, .general_purpose, cg)) break; + } else break; + const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg); + const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg); + const result: MCValue = .{ .register_pair = .{ + first_temp_tracking.short.register.to64(), + second_temp_tracking.short.register.to64(), + } }; + const result_temp_index = cg.next_temp_index; + const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; + assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking)); + assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking)); + result_temp_index.tracking(cg).* = .init(result); + cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; + cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); + first_temp.* = result_temp; + second_temp.* = result_temp; + } + + fn withOverflow(temp: *Temp, overflow_temp: *Temp, cg: *CodeGen) InnerError!void { + const temp_index = temp.unwrap(cg).temp; + const temp_tracking = temp_index.tracking(cg); + const overflow_temp_tracking = overflow_temp.unwrap(cg).temp.tracking(cg); + switch (temp_tracking.short) { + .register => |reg| switch (overflow_temp_tracking.short) { + .eflags => |overflow_cc| { + const result_temp_index = cg.next_temp_index; + const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; + assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); + assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); + result_temp_index.tracking(cg).* = .init(.{ .register_overflow = .{ .reg = reg, .eflags = overflow_cc } }); + cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; + cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); + temp.* = result_temp; + overflow_temp.* = result_temp; + return; + }, + .register => |overflow_reg| { + const result_temp_index = cg.next_temp_index; + const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; + assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); + assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); + result_temp_index.tracking(cg).* = .init(.{ .register_pair = .{ reg, overflow_reg } }); + cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; + cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); + temp.* = result_temp; + overflow_temp.* = result_temp; + return; + }, + else => {}, + }, + .load_frame => { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + const field_tys = ip.indexToKey(temp_index.typeOf(cg).toIntern()).tuple_type.types.get(ip); + try temp.write(overflow_temp, .{ .disp = @intCast(Type.fromInterned(field_tys[0]).abiSize(zcu)) }, cg); + try overflow_temp.die(cg); + overflow_temp.* = temp.*; + return; + }, + else => {}, + } + std.debug.panic("{s}: {f} {f}\n", .{ @src().fn_name, temp_tracking, overflow_temp_tracking }); + } + + fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void { + assert(info.scalar != .none); + const mcv = &temp.unwrap(cg).temp.tracking(cg).short; + const reg = mcv.register; + mcv.* = .{ .register_mask = .{ .reg = reg, .info = info } }; + } + + fn toLea(temp: *Temp, cg: *CodeGen) InnerError!bool { + switch (temp.tracking(cg).short) { + .none, + .unreach, + .dead, + .undef, + .eflags, + .register_pair, + .register_triple, + .register_quadruple, + .register_overflow, + .register_mask, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, // not a valid pointer + .immediate, + .register, + .register_offset, + .lea_frame, + => return false, + .memory, + .indirect, + .load_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => return temp.toRegClass(true, .general_purpose, cg), + } + } + + fn toMemory(temp: *Temp, mut: bool, cg: *CodeGen) InnerError!bool { + const temp_tracking = temp.tracking(cg); + if ((!mut or temp.isMut(cg)) and temp_tracking.short.isMemory()) return false; + const new_temp_index = cg.next_temp_index; + const ty = temp.typeOf(cg); + cg.temp_type[@intFromEnum(new_temp_index)] = ty; + const new_frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu)); + try cg.genSetMem(.{ .frame = new_frame_index }, 0, ty, temp_tracking.short, .{}); + new_temp_index.tracking(cg).* = .init(.{ .load_frame = .{ .index = new_frame_index } }); + try temp.die(cg); + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + // hack around linker relocation bugs + fn toBase(temp: *Temp, mut: bool, cg: *CodeGen) InnerError!bool { + const temp_tracking = temp.tracking(cg); + if ((!mut or temp.isMut(cg)) and temp_tracking.short.isBase()) return false; + if (try temp.toMemory(mut, cg)) return true; + const new_temp_index = cg.next_temp_index; + cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg); + const new_reg = + try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{}); + new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } }); + try temp.die(cg); + cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + const AccessOptions = struct { + disp: i32 = 0, + safe: bool = false, + }; + + fn load(ptr: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) InnerError!Temp { + const val = try cg.tempAlloc(val_ty); + try ptr.toOffset(opts.disp, cg); + while (try ptr.toLea(cg)) {} + const val_mcv = val.tracking(cg).short; + switch (val_mcv) { + else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), + .register => |val_reg| try ptr.loadReg(val_ty, registerAlias( + val_reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + ), cg), + inline .register_pair, + .register_triple, + .register_quadruple, + => |val_regs| for (val_regs) |val_reg| { + try ptr.loadReg(val_ty, val_reg, cg); + try ptr.toOffset(@intCast(@divExact(val_reg.size().bitSize(cg.target), 8)), cg); + while (try ptr.toLea(cg)) {} + }, + .register_offset => |val_reg_off| switch (val_reg_off.off) { + 0 => try ptr.loadReg(val_ty, registerAlias( + val_reg_off.reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + ), cg), + else => unreachable, + }, + .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + var val_ptr = try cg.tempInit(.usize, val_mcv.address()); + var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); + try val_ptr.memcpy(ptr, &len, cg); + try val_ptr.die(cg); + try len.die(cg); + }, + } + return val; + } + + fn store(ptr: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) InnerError!void { + const val_ty = val.typeOf(cg); + try ptr.toOffset(opts.disp, cg); + while (try ptr.toLea(cg)) {} + val_to_gpr: while (true) : (while (try ptr.toLea(cg) or + try val.toRegClass(false, .general_purpose, cg)) + {}) { + const val_mcv = val.tracking(cg).short; + switch (val_mcv) { + else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), + .undef => if (opts.safe) { + var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); + var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); + try ptr.memset(&pat, &len, cg); + try pat.die(cg); + try len.die(cg); + }, + .immediate => |val_imm| { + const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31| + .u(val_uimm31) + else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32| + .s(val_simm32) + else + continue :val_to_gpr; + // hack around linker relocation bugs + switch (ptr.tracking(cg).short) { + else => {}, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, + } + try cg.asmMemoryImmediate( + .{ ._, .mov }, + try ptr.tracking(cg).short.deref().mem(cg, .{ + .size = cg.memSize(val_ty), + }), + val_op, + ); + }, + .eflags => |cc| { + // hack around linker relocation bugs + switch (ptr.tracking(cg).short) { + else => {}, + .lea_nav, + .lea_uav, + .lea_lazy_sym, + .lea_extern_func, + => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, + } + try cg.asmSetccMemory( + cc, + try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }), + ); + }, + .register => |val_reg| try ptr.storeRegs(val_ty, &.{registerAlias( + val_reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + )}, cg), + inline .register_pair, + .register_triple, + .register_quadruple, + => |val_regs| try ptr.storeRegs(val_ty, &val_regs, cg), + .register_offset => |val_reg_off| switch (val_reg_off.off) { + 0 => try ptr.storeRegs(val_ty, &.{registerAlias( + val_reg_off.reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + )}, cg), + else => continue :val_to_gpr, + }, + .register_overflow => |val_reg_ov| { + const ip = &cg.pt.zcu.intern_pool; + const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) { + .tuple_type => |tuple_type| { + const tuple_field_types = tuple_type.types.get(ip); + assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type); + break :first_ty tuple_field_types[0]; + }, + .opt_type => |opt_child| { + assert(!val_ty.optionalReprIsPayload(cg.pt.zcu)); + break :first_ty opt_child; + }, + else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }), + }); + const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu)); + try ptr.storeRegs(first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg); + try ptr.toOffset(first_size, cg); + try cg.asmSetccMemory( + val_reg_ov.eflags, + try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }), + ); + }, + .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr, + .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + var val_ptr = try cg.tempInit(.usize, val_mcv.address()); + var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); + try ptr.memcpy(&val_ptr, &len, cg); + try val_ptr.die(cg); + try len.die(cg); + }, + } + break; + } + } + + fn read(src: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) InnerError!Temp { + var val = try cg.tempAlloc(val_ty); + while (try src.toBase(false, cg)) {} + try src.readTo(val_ty, val.tracking(cg).short, opts, cg); + return val; + } + + fn readTo(src: *Temp, val_ty: Type, val_mcv: MCValue, opts: AccessOptions, cg: *CodeGen) InnerError!void { + switch (val_mcv) { + else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), + .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias( + val_reg, + @intCast(cg.unalignedSize(val_ty)), + ), cg), + inline .register_pair, .register_triple, .register_quadruple => |val_regs| { + var disp = opts.disp; + for (val_regs) |val_reg| { + try src.readReg(disp, val_ty, val_reg, cg); + disp += @intCast(@divExact(val_reg.size().bitSize(cg.target), 8)); + } + }, + .register_offset => |val_reg_off| switch (val_reg_off.off) { + 0 => try src.readReg(opts.disp, val_ty, registerAlias( + val_reg_off.reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + ), cg), + else => unreachable, + }, + .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + var val_ptr = try cg.tempInit(.usize, val_mcv.address()); + var src_ptr = + try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp)); + var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); + try val_ptr.memcpy(&src_ptr, &len, cg); + try val_ptr.die(cg); + try src_ptr.die(cg); + try len.die(cg); + }, + } + } + + fn write(dst: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) InnerError!void { + const val_ty = val.typeOf(cg); + while (try dst.toBase(false, cg)) {} + val_to_gpr: while (true) : (while (try dst.toBase(false, cg) or + try val.toRegClass(false, .general_purpose, cg)) + {}) { + const val_mcv = val.tracking(cg).short; + switch (val_mcv) { + else => |mcv| std.debug.panic("{s}: {f}\n", .{ @src().fn_name, mcv }), + .none => {}, + .undef => if (opts.safe) { + var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); + var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); + var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); + try dst_ptr.memset(&pat, &len, cg); + try dst_ptr.die(cg); + try pat.die(cg); + try len.die(cg); + }, + .immediate => |val_imm| { + const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31| + .u(val_uimm31) + else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32| + .s(val_simm32) + else + continue :val_to_gpr; + try cg.asmMemoryImmediate( + .{ ._, .mov }, + try dst.tracking(cg).short.mem(cg, .{ + .size = cg.memSize(val_ty), + .disp = opts.disp, + }), + val_op, + ); + }, + .eflags => |cc| try cg.asmSetccMemory( + cc, + try dst.tracking(cg).short.mem(cg, .{ + .size = .byte, + .disp = opts.disp, + }), + ), + .register => |val_reg| try dst.writeReg(opts.disp, val_ty, registerAlias( + val_reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + ), cg), + inline .register_pair, + .register_triple, + .register_quadruple, + => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg), + .register_offset => |val_reg_off| switch (val_reg_off.off) { + 0 => try dst.writeReg(opts.disp, val_ty, registerAlias( + val_reg_off.reg, + @intCast(val_ty.abiSize(cg.pt.zcu)), + ), cg), + else => continue :val_to_gpr, + }, + .register_overflow => |val_reg_ov| { + const ip = &cg.pt.zcu.intern_pool; + const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) { + .tuple_type => |tuple_type| { + const tuple_field_types = tuple_type.types.get(ip); + assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type); + break :first_ty tuple_field_types[0]; + }, + .opt_type => |opt_child| { + assert(!val_ty.optionalReprIsPayload(cg.pt.zcu)); + break :first_ty opt_child; + }, + else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }), + }); + const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu)); + try dst.writeReg(opts.disp, first_ty, registerAlias(val_reg_ov.reg, first_size), cg); + try cg.asmSetccMemory( + val_reg_ov.eflags, + try dst.tracking(cg).short.mem(cg, .{ + .size = .byte, + .disp = opts.disp + first_size, + }), + ); + }, + .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr, + .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => { + var dst_ptr = + try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); + var val_ptr = try cg.tempInit(.usize, val_mcv.address()); + var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) }); + try dst_ptr.memcpy(&val_ptr, &len, cg); + try dst_ptr.die(cg); + try val_ptr.die(cg); + try len.die(cg); + }, + } + break; + } + } + + fn loadReg(ptr: *Temp, dst_ty: Type, dst_reg: Register, cg: *CodeGen) InnerError!void { + const dst_rc = dst_reg.class(); + const strat = try cg.moveStrategy(dst_ty, dst_rc, false); + // hack around linker relocation bugs + switch (ptr.tracking(cg).short) { + else => {}, + .lea_nav, .lea_uav, .lea_lazy_sym => if (dst_rc != .general_purpose) + while (try ptr.toRegClass(false, .general_purpose, cg)) {}, + } + try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{ + .size = cg.memSize(dst_ty), + })); + } + + fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + var part_disp: u31 = 0; + var deferred_disp: u31 = 0; + var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu)); + for (src_regs, 0..) |src_reg, part_index| { + const part_ty: Type = if (src_regs.len == 1) + src_ty + else if (cg.intInfo(src_ty)) |int_info| part_ty: { + assert(src_regs.len == std.math.divCeil(u16, int_info.bits, 64) catch unreachable); + break :part_ty .u64; + } else part_ty: switch (ip.indexToKey(src_ty.toIntern())) { + else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, src_ty.fmt(cg.pt) }), + .ptr_type => |ptr_info| { + assert(ptr_info.flags.size == .slice); + assert(src_regs.len == 2); + break :part_ty .usize; + }, + .array_type => { + assert(src_regs.len - part_index == std.math.divCeil(u32, src_abi_size, 8) catch unreachable); + break :part_ty try cg.pt.intType(.unsigned, @as(u16, 8) * @min(src_abi_size, 8)); + }, + .opt_type => |opt_child| switch (ip.indexToKey(opt_child)) { + else => std.debug.panic("{s}: {f}\n", .{ @src().fn_name, src_ty.fmt(cg.pt) }), + .ptr_type => |ptr_info| { + assert(ptr_info.flags.size == .slice); + assert(src_regs.len == 2); + break :part_ty .usize; + }, + }, + .struct_type => { + assert(src_regs.len - part_index == std.math.divCeil(u32, src_abi_size, 8) catch unreachable); + break :part_ty switch (src_abi_size) { + 0, 3, 5...7 => unreachable, + 1 => .u8, + 2 => .u16, + 4 => .u32, + else => .u64, + }; + }, + .tuple_type => |tuple_type| { + assert(tuple_type.types.len == src_regs.len); + break :part_ty .fromInterned(tuple_type.types.get(ip)[part_index]); + }, + }; + const part_size: u31 = @intCast(part_ty.abiSize(zcu)); + const src_rc = src_reg.class(); + if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) { + // hack around linker relocation bugs + switch (ptr.tracking(cg).short) { + else => {}, + .lea_nav, .lea_uav, .lea_lazy_sym => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, + } + const strat = try cg.moveStrategy(part_ty, src_rc, false); + try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{ + .size = switch (src_rc) { + else => .fromBitSize(8 * part_size), + .x87 => switch (abi.classifySystemV(src_ty, zcu, cg.target, .other)[part_index]) { + else => unreachable, + .float => .dword, + .float_combine, .sse => .qword, + .x87 => .tbyte, + }, + }, + .disp = part_disp, + }), registerAlias(src_reg, part_size)); + } else { + const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size); + const frame_index = try cg.allocFrameIndex(.init(.{ + .size = frame_size, + .alignment = .fromNonzeroByteUnits(frame_size), + })); + const strat = try cg.moveStrategy(part_ty, src_rc, true); + try strat.write(cg, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, + }, registerAlias(src_reg, frame_size)); + try ptr.toOffset(deferred_disp, cg); + deferred_disp = 0; + var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } }); + var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size }); + try ptr.memcpy(&src_ptr, &len, cg); + try src_ptr.die(cg); + try len.die(cg); + } + part_disp += part_size; + deferred_disp += part_size; + src_abi_size -= part_size; + } + } + + fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) InnerError!void { + const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false); + try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{ + .size = cg.memSize(dst_ty), + .disp = disp, + })); + } + + fn writeReg(dst: Temp, disp: i32, src_ty: Type, src_reg: Register, cg: *CodeGen) InnerError!void { + const src_abi_size: u31 = @intCast(src_ty.abiSize(cg.pt.zcu)); + const src_rc = src_reg.class(); + if (src_rc == .x87 or std.math.isPowerOfTwo(src_abi_size)) { + const strat = try cg.moveStrategy(src_ty, src_rc, false); + try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{ + .size = cg.memSize(src_ty), + .disp = disp, + }), registerAlias(src_reg, src_abi_size)); + } else { + const frame_size = std.math.ceilPowerOfTwoAssert(u32, src_abi_size); + const frame_index = try cg.allocFrameIndex(.init(.{ + .size = frame_size, + .alignment = .fromNonzeroByteUnits(frame_size), + })); + const strat = try cg.moveStrategy(src_ty, src_rc, true); + try strat.write(cg, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, + }, registerAlias(src_reg, frame_size)); + var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address()); + try dst_ptr.toOffset(disp, cg); + var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } }); + var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size }); + try dst_ptr.memcpy(&src_ptr, &len, cg); + try dst_ptr.die(cg); + try src_ptr.die(cg); + try len.die(cg); + } + } + + fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { + const zcu = cg.pt.zcu; + const classes = std.mem.sliceTo(&abi.classifySystemV(src_ty, zcu, cg.target, .other), .none); + var next_class_index: u4 = 0; + var part_disp = disp; + var remaining_abi_size = src_ty.abiSize(zcu); + for (src_regs) |src_reg| { + const class_index = next_class_index; + const class = classes[class_index]; + next_class_index = @intCast(switch (class) { + .integer, .memory, .float, .float_combine => class_index + 1, + .sse => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.sseup}) orelse classes.len, + .x87 => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.x87up}) orelse classes.len, + .sseup, .x87up, .none, .win_i128, .integer_per_element => unreachable, + }); + const part_size = switch (class) { + .integer, .sse, .memory => @min(8 * @as(u7, next_class_index - class_index), remaining_abi_size), + .x87 => 16, + .float => 4, + .float_combine => 8, + .sseup, .x87up, .none, .win_i128, .integer_per_element => unreachable, + }; + try dst.writeReg(part_disp, switch (class) { + .integer => .u64, + .sse => switch (part_size) { + else => unreachable, + 8 => .f64, + 16 => .vector_2_f64, + 32 => .vector_4_f64, + }, + .x87 => .f80, + .float => .f32, + .float_combine => .vector_2_f32, + .sseup, .x87up, .memory, .none, .win_i128, .integer_per_element => unreachable, + }, src_reg, cg); + part_disp += part_size; + remaining_abi_size -= part_size; + } + assert(next_class_index == classes.len); + } + + fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void { + while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| { + if (try temp.toReg(reg, cg)) break; + } else break; + try cg.asmOpOnly(.{ .@"rep _sb", .mov }); + } + + fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) InnerError!void { + while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) |temp, reg| { + if (try temp.toReg(reg, cg)) break; + } else break; + try cg.asmOpOnly(.{ .@"rep _sb", .sto }); + } + + fn wrapInt(temp: *Temp, cg: *CodeGen) Select.Error!void { + var ops: [1]Temp = .{temp.*}; + var res: [1]Temp = undefined; + try cg.select(&res, &.{temp.typeOf(cg)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_int = 8 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 16 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_int = 32 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_int = 64 }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._l, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .none, .none } }, + .{ .src = .{ .to_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .src0q, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_gpr, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .tmp0q, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_remainder_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .memad(.dst0q, .add_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._l, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 8 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 8 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_b, .add, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_b, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .byte, .is = 8 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_b, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_b, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1b, .uia(8, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1b, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 16 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 16 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .sll, .dst0x, .src0x, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .dst0x, .dst0x, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_w, .sll, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_w, .sra, .dst0x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_w, .sll, .dst0y, .src0y, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .dst0y, .dst0y, .uia(16, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .word, .is = 16 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sll, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .tmp1y, .tmp1y, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_w, .sll, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_w, .sra, .tmp1x, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_w, .sll, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_w, .sra, .tmp1x, .uia(16, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1w, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .sll, .dst0x, .src0x, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .dst0x, .dst0x, .uia(32, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, .p_d, .sll, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_d, .sra, .dst0x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, .vp_d, .sll, .dst0y, .src0y, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .dst0y, .dst0y, .uia(32, .dst0, .sub_bit_size), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sll, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .tmp1y, .tmp1y, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .vp_d, .sll, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .vp_d, .sra, .tmp1x, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_d, .sll, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, .p_d, .sra, .tmp1x, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .dword, .is = 32 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1d, .uia(32, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2d, .memia(.src0d, .tmp0, .add_unaligned_size), .tmp1d, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp2d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .@"and", .tmp1d, .sa(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_q, .add, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + .{ ._, .vp_, .xor, .dst0x, .dst0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.tmp0x), ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .p_q, .add, .dst0x, .lea(.tmp0x), ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .p_, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .@"and", .dst0x, .lea(.tmp0x), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp2), ._, ._ }, + .{ ._, .vp_q, .add, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + .{ ._, .vp_, .xor, .dst0y, .dst0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.tmp0y), ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .add, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp2y, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1y, .lea(.tmp0y), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp2y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .vp_q, .add, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp2x, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .{ .kind = .{ .smin_mem = .{ .ref = .dst0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp4), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.tmp5), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .p_q, .add, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .kind = .{ .umax_mem = .{ .ref = .dst0, .to_signedness = .unsigned } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .lea(.tmp0x), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp2x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .multiple_scalar_exact_int = .{ .of = .qword, .is = 64 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .dst0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ }, + .{ .@"0:", ._, .bzhi, .tmp2q, .memia(.src0q, .tmp0, .add_unaligned_size), .tmp1q, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.dst0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .ua(.dst0, .add_umax), ._, ._ }, + .{ ._, ._, .@"and", .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .xword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_exact_int = .{ .of = .yword, .is = 128 } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .scalar_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .@"and", .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .si(0), ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ }, + .{ ._, ._l, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_umax), ._, ._ }, + .{ .@"0:", ._, .@"and", .memi(.dst0q, .tmp0), .tmp1q, ._, ._ }, + .{ ._, ._, .sub, .tmp0d, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }); + for (ops) |op| for (res) |r| { + if (op.index == r.index) break; + } else try op.die(cg); + temp.* = res[0]; + } + + /// Supports any `op` using `cg.intInfo(lhs.typeOf(cg)).?.signedness` as the signedness. + /// Returns `error.SelectFailed` when `cg.intInfo(lhs.typeOf(cg)) == null`. + fn cmpInts(lhs: *Temp, op: std.math.CompareOperator, rhs: *Temp, cg: *CodeGen) Select.Error!Temp { + var ops: [2]Temp = .{ lhs.*, rhs.* }; + var res: [1]Temp = undefined; + switch (op) { + .lt, .lte, .gte, .gt => { + const commute = switch (op) { + .lt, .gte => false, + .lte, .gt => true, + else => unreachable, + }; + if (commute) std.mem.swap(Temp, &ops[0], &ops[1]); + try cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .g }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm8, .none } }, + .{ .src = .{ .to_gpr, .imm8, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .l }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .a }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm8, .none } }, + .{ .src = .{ .to_gpr, .imm8, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .b }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .g }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm16, .none } }, + .{ .src = .{ .to_gpr, .imm16, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .l }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .a }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm16, .none } }, + .{ .src = .{ .to_gpr, .imm16, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .b }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .g }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm32, .none } }, + .{ .src = .{ .to_gpr, .imm32, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .l }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .a }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm32, .none } }, + .{ .src = .{ .to_gpr, .imm32, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .b }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .g }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .simm32, .none } }, + .{ .src = .{ .to_gpr, .simm32, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .l }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .cc = .a }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .simm32, .none } }, + .{ .src = .{ .to_gpr, .simm32, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .b }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .l }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .b }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .l }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_4), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"4", .tmp0, .add_size, -4), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"4", .tmp0, .add_size, -4), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -4), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -4), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .b }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"4", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }); + if (commute) std.mem.swap(Temp, &ops[0], &ops[1]); + }, + .eq, .neq => { + try cg.select(&res, &.{.bool}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm8, .none } }, + .{ .src = .{ .imm8, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm16, .none } }, + .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .imm32, .none } }, + .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .simm32, .none } }, + .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .cc = .e }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_mm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_mm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mm, .to_mm, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ }, + .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ }, + .{ ._, ._, .cmp, .tmp0b, .si(-1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_xmm, .to_xmm, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ }, + .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .fast_imm16, null, null }, + .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ }, + .{ ._, ._, .cmp, .tmp0w, .si(-1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_xmm, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_xmm, .to_xmm, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .si(std.math.maxInt(u16)), ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .mem, .none } }, + .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_ymm, .to_ymm, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ }, + .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_ymm, .mem, .none } }, + .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_ymm, .to_ymm, .none } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ }, + .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .yword, .is = .xword } }, + .{ .remainder_int = .{ .of = .yword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .memiad(.src1y, .tmp0, .add_size, -16), ._ }, + .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp1x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .memad(.src1x, .add_size, -16), ._ }, + .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .yword, .is = .xword } }, + .{ .remainder_int = .{ .of = .yword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .movu, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .memiad(.src1y, .tmp0, .add_size, -16), ._ }, + .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, .v_pd, .mova, .tmp1x, .memad(.src0x, .add_size, -16), ._, ._ }, + .{ ._, .v_pd, .xor, .tmp1x, .tmp1x, .memad(.src1x, .add_size, -16), ._ }, + .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .fast_imm16, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp1x, ._, ._ }, + .{ ._, ._, .cmp, .tmp2w, .si(-1), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .sse } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp1x, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .si(std.math.maxInt(u16)), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .rc = .mmx } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpeq, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp2d, .tmp1q, ._, ._ }, + .{ ._, ._, .cmp, .tmp2b, .si(-1), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .remainder_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .cc = .z }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }); + }, + } + if (switch (op) { + .lt, .gt, .eq => false, + .lte, .gte, .neq => true, + }) { + const cc = &res[0].unwrap(cg).temp.tracking(cg).short.eflags; + cc.* = cc.negate(); + } + lhs.*, rhs.* = ops; + return res[0]; + } + + fn divTruncInts(lhs: *Temp, rhs: *Temp, cg: *CodeGen) Select.Error!Temp { + var ops: [2]Temp = .{ lhs.*, rhs.* }; + var res: [1]Temp = undefined; + try cg.select(&res, &.{lhs.typeOf(cg)}, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 1 }, .{ .exact_signed_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .any, .any, .none } }, + }, + .dst_temps = .{ .{ .imm = 0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 1 }, .{ .exact_unsigned_int = 1 }, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .any, .none } }, + .{ .src = .{ .to_mut_gpr, .any, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{} }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .al }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movsx, .dst0d, .src0b, ._, ._ }, + .{ ._, .i_, .div, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .mem, .mem, .none } }, + .{ .src = .{ .to_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_gpr, .none } }, + .{ .src = .{ .to_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .reg = .al }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .movzx, .dst0d, .src0b, ._, ._ }, + .{ ._, ._, .div, .src1b, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1w, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .div, .src1w, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1d, ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .div, .src1d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .src1q, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } }, + .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0q, .tmp0q, ._, ._ }, + .{ ._, ._, .div, .src1q, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, + .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } }, + .none, + } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._, .call, .tmp2d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .mem(.src1), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .reg = .al } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .reg = .ax } }, + .{ .type = .i16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cwd, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u16, .kind = .{ .reg = .dx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .div, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .i32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cdq, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .eax } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .div, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .reg = .rax } }, + .{ .type = .i64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .cqo, ._, ._, ._, ._ }, + .{ ._, .i_, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .reg = .rax } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ ._, ._, .div, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .i64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divti3" } }, + .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .sysv64, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .u64, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, + .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4q, .memiad(.src1q, .tmp0, .add_unaligned_size, 8), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp6q, ._, ._ }, + .{ ._, ._, .mov, .memiad(.dst0q, .tmp0, .add_unaligned_size, 8), .tmp3q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, + .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, + .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_cc_abi = .win64, + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivti3" } }, + .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__divei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mem, .to_mut_mem, .none } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } }, + .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 3, .at = 3 } } }, + .{ .type = .usize, .kind = .{ .extern_func = "__udivei4" } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .lea, .tmp3p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + } }); + lhs.*, rhs.* = ops; + return res[0]; + } + + fn finish( + temp: Temp, + inst: Air.Inst.Index, + op_refs: []const Air.Inst.Ref, + op_temps: []const Temp, + cg: *CodeGen, + ) InnerError!void { + const tomb_bits = cg.liveness.getTombBits(inst); + for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { + if (op_temp.index == temp.index) continue; + if (op_temp.tracking(cg).short != .dead) try op_temp.die(cg); + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (cg.reused_operands.isSet(op_index)) continue; + try cg.processDeath(op_ref.toIndexAllowNone() orelse continue, .{}); + } + if (cg.liveness.isUnused(inst)) try temp.die(cg) else { + switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => { + const temp_mcv = temp.tracking(cg).short; + const result = result: switch (temp_mcv) { + .none, + .unreach, + .dead, + .indirect_load_frame, + .elementwise_args, + .reserved_frame, + .air_ref, + => unreachable, + .undef, .immediate, .lea_frame => temp_mcv, + .eflags, + .register, + .register_pair, + .register_triple, + .register_quadruple, + .register_offset, + .register_overflow, + .register_mask, + .memory, + .indirect, + .load_frame, + .load_nav, + .lea_nav, + .load_uav, + .lea_uav, + .load_lazy_sym, + .lea_lazy_sym, + .load_extern_func, + .lea_extern_func, + => { + const result = try cg.allocRegOrMem(inst, true); + try cg.genCopy(cg.typeOfIndex(inst), result, temp_mcv, .{}); + break :result result; + }, + }; + tracking_log.debug("{f} => {f} (birth)", .{ inst, result }); + cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result)); + }, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(cg); + tracking_log.debug("{f} => {f} (birth)", .{ inst, temp_tracking.short }); + cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(temp_tracking.short)); + assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking)); + }, + } + } + for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { + if (op_temp.index != temp.index) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (cg.reused_operands.isSet(op_index)) continue; + try cg.processDeath(op_ref.toIndexAllowNone() orelse continue, .{}); + } + } + + fn die(temp: Temp, cg: *CodeGen) InnerError!void { + switch (temp.unwrap(cg)) { + .ref, .err_ret_trace => {}, + .temp => |temp_index| try temp_index.tracking(cg).die(cg, temp_index.toIndex(), .{}), + } + } + + const Index = enum(u5) { + _, + + fn toIndex(index: Index) Air.Inst.Index { + return .fromTargetIndex(@intFromEnum(index)); + } + + fn fromIndex(index: Air.Inst.Index) Index { + return @enumFromInt(index.toTargetIndex()); + } + + fn tracking(index: Index, cg: *CodeGen) *InstTracking { + return &cg.inst_tracking.values()[@intFromEnum(index)]; + } + + fn isValid(index: Index, cg: *CodeGen) bool { + return @intFromEnum(index) < @intFromEnum(cg.next_temp_index) and + index.tracking(cg).short != .dead; + } + + fn typeOf(index: Index, cg: *CodeGen) Type { + assert(index.isValid(cg)); + return cg.temp_type[@intFromEnum(index)]; + } + + const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type); + const Set = std.StaticBitSet(max); + const SafetySet = if (std.debug.runtime_safety) Set else struct { + inline fn initEmpty() @This() { + return .{}; + } + + inline fn isSet(_: @This(), index: usize) bool { + assert(index < max); + return true; + } + + inline fn set(_: @This(), index: usize) void { + assert(index < max); + } + + inline fn eql(_: @This(), _: @This()) bool { + return true; + } + }; + }; +}; + +fn resetTemps(cg: *CodeGen, from_index: Temp.Index) InnerError!void { + var any_valid = false; + for (@intFromEnum(from_index)..@intFromEnum(cg.next_temp_index)) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + if (temp.isValid(cg)) { + any_valid = true; + tracking_log.err("failed to kill {f}: {f}", .{ + temp.toIndex(), + cg.temp_type[temp_index].fmt(cg.pt), + }); + } + cg.temp_type[temp_index] = undefined; + } + if (any_valid) return cg.fail("failed to kill all temps", .{}); + cg.next_temp_index = from_index; +} + +fn reuseTemp( + cg: *CodeGen, + new_inst: Air.Inst.Index, + old_inst: Air.Inst.Index, + tracking: *InstTracking, +) bool { + switch (tracking.short) { + .register, + .register_pair, + .register_offset, + .register_overflow, + .register_mask, + .indirect, + => for (tracking.short.getRegs()) |tracked_reg| { + if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { + cg.register_manager.registers[tracked_index] = new_inst; + } + }, + .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, + else => {}, + } + switch (tracking.short) { + .eflags, .register_overflow => cg.eflags_inst = new_inst, + else => {}, + } + tracking.reuse(cg, new_inst, old_inst); + return true; +} + +fn tempAlloc(cg: *CodeGen, ty: Type) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true), + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempAllocReg(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + .{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), rs) }, + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + .{ .register_pair = try cg.register_manager.allocRegs(2, @splat(temp_index.toIndex()), rs) }, + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempAllocMem(cg: *CodeGen, ty: Type) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init( + try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false), + ); + cg.temp_type[@intFromEnum(temp_index)] = ty; + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempInit(cg: *CodeGen, ty: Type, value: MCValue) InnerError!Temp { + const temp_index = cg.next_temp_index; + temp_index.tracking(cg).* = .init(value); + cg.temp_type[@intFromEnum(temp_index)] = ty; + try cg.getValue(value, temp_index.toIndex()); + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempFromValue(cg: *CodeGen, value: Value) InnerError!Temp { + return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerValue(value)); +} + +fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp { + return cg.tempMemFromAlignedValue(.none, value); +} + +fn tempMemFromAlignedValue(cg: *CodeGen, alignment: InternPool.Alignment, value: Value) InnerError!Temp { + const ty = value.typeOf(cg.pt.zcu); + return cg.tempInit(ty, .{ .load_uav = .{ + .val = value.toIntern(), + .orig_ty = (try cg.pt.ptrType(.{ + .child = ty.toIntern(), + .flags = .{ + .is_const = true, + .alignment = alignment, + }, + })).toIntern(), + } }); +} + +fn tempFromOperand(cg: *CodeGen, op_ref: Air.Inst.Ref, op_dies: bool) InnerError!Temp { + if (op_dies) { + const temp_index = cg.next_temp_index; + const temp: Temp = .{ .index = temp_index.toIndex() }; + const op_inst = op_ref.toIndex().?; + const tracking = cg.getResolvedInstValue(op_inst); + temp_index.tracking(cg).* = tracking.*; + if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; + cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); + cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return temp; + } + + if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; + return cg.tempFromValue(.fromInterned(op_ref.toInterned().?)); +} + +fn tempsFromOperandsInner( + cg: *CodeGen, + inst: Air.Inst.Index, + op_temps: []Temp, + op_refs: []const Air.Inst.Ref, +) InnerError!void { + for (op_temps, 0.., op_refs) |*op_temp, op_index, op_ref| op_temp.* = try cg.tempFromOperand(op_ref, for (op_refs[0..op_index]) |prev_op_ref| { + if (op_ref == prev_op_ref) break false; + } else cg.liveness.operandDies(inst, @intCast(op_index))); +} + +inline fn tempsFromOperands( + cg: *CodeGen, + inst: Air.Inst.Index, + op_refs: anytype, +) InnerError![op_refs.len]Temp { + var op_temps: [op_refs.len]Temp = undefined; + try cg.tempsFromOperandsInner(inst, &op_temps, &op_refs); + return op_temps; +} + +const Operand = union(enum) { + none, + reg: Register, + mem: Memory, + imm: Immediate, + inst: Mir.Inst.Index, +}; + +const Select = struct { + cg: *CodeGen, + types: [@intFromEnum(Select.Operand.Ref.none)]Type, + temps: [@intFromEnum(Select.Operand.Ref.none)]Temp, + labels: [@intFromEnum(Label._)]struct { + backward: ?Mir.Inst.Index, + forward: [3]?Mir.Inst.Index, + }, + top: u3, + + const Error = InnerError || error{SelectFailed}; + + fn emitLabel(s: *Select, label_index: Label) void { + assert(@intFromEnum(label_index) < @intFromEnum(Label._)); + const label = &s.labels[@intFromEnum(label_index)]; + for (&label.forward) |*reloc| { + if (reloc.*) |r| s.cg.performReloc(r); + reloc.* = null; + } + label.backward = @intCast(s.cg.mir_instructions.len); + } + + fn emit(s: *Select, inst: Instruction) InnerError!void { + const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] }; + pseudo: { + switch (inst[0]) { + .@"0:", .@"1:", .@"2:", .@"3:", .@"4:" => |label| s.emitLabel(label), + ._ => {}, + .pseudo => break :pseudo, + } + var mir_ops: [4]CodeGen.Operand = undefined; + inline for (&mir_ops, 3..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s); + s.cg.asmOps(mir_tag, mir_ops) catch |err| switch (err) { + error.InvalidInstruction => { + const fixes = @tagName(mir_tag[0]); + const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; + return s.cg.fail("invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'", .{ + fixes[0..fixes_blank], + @tagName(mir_tag[1]), + fixes[fixes_blank + 1 ..], + @tagName(mir_ops[0]), + @tagName(mir_ops[1]), + @tagName(mir_ops[2]), + @tagName(mir_ops[3]), + }); + }, + else => |e| return e, + }; + } + switch (mir_tag[0]) { + .f_ => switch (mir_tag[1]) { + .@"2xm1", + .abs, + .add, + .chs, + .clex, + .com, + .comi, + .cos, + .div, + .divr, + .free, + .mul, + .nop, + .prem, + .rndint, + .scale, + .sin, + .sqrt, + .st, + .sub, + .subr, + .tst, + .ucom, + .ucomi, + .wait, + .xam, + .xch, + => {}, + .init, .save => s.top = 0, + .ld, .ptan, .sincos, .xtract => s.top -%= 1, + .patan, .yl2x => s.top +%= 1, + .rstor => unreachable, + else => unreachable, + }, + .f_1 => switch (mir_tag[1]) { + .ld => s.top -%= 1, + .prem => {}, + else => unreachable, + }, + .f_b, .f_be, .f_e, .f_nb, .f_nbe, .f_ne, .f_nu, .f_u => switch (mir_tag[1]) { + .cmov => {}, + else => unreachable, + }, + .f_cw, .f_env, .f_sw => switch (mir_tag[1]) { + .ld, .st => {}, + else => unreachable, + }, + .f_p1 => switch (mir_tag[1]) { + .yl2x => s.top +%= 1, + else => unreachable, + }, + .f_cstp => switch (mir_tag[1]) { + .de => s.top -%= 1, + .in => s.top +%= 1, + else => unreachable, + }, + .f_l2e, .f_l2t, .f_lg2, .f_ln2, .f_pi, .f_z => switch (mir_tag[1]) { + .ld => s.top -%= 1, + else => unreachable, + }, + .f_p => switch (mir_tag[1]) { + .add, .com, .comi, .div, .divr, .mul, .st, .sub, .subr, .ucom, .ucomi => s.top +%= 1, + else => { + const fixes = @tagName(mir_tag[0]); + const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; + std.debug.panic("{s}: {s}{s}{s}\n", .{ + @src().fn_name, + fixes[0..fixes_blank], + @tagName(mir_tag[1]), + fixes[fixes_blank + 1 ..], + }); + }, + }, + .f_pp => switch (mir_tag[1]) { + .com, .ucom => s.top +%= 2, + else => unreachable, + }, + .fb_ => switch (mir_tag[1]) { + .ld => s.top -%= 1, + else => unreachable, + }, + .fb_p => switch (mir_tag[1]) { + .st => s.top +%= 1, + else => unreachable, + }, + .fi_ => switch (mir_tag[1]) { + .add, .com, .div, .divr, .mul, .st, .stt, .sub, .subr => {}, + .ld => s.top -%= 1, + else => unreachable, + }, + .fi_p => switch (mir_tag[1]) { + .com, .st, .stt => s.top +%= 1, + else => unreachable, + }, + .fn_ => switch (mir_tag[1]) { + .clex => {}, + .init, .save => s.top = 0, + else => unreachable, + }, + .fn_cw, .fn_env, .fn_sw => switch (mir_tag[1]) { + .st => {}, + else => unreachable, + }, + .fx_ => switch (mir_tag[1]) { + .rstor => unreachable, + .save => {}, + else => unreachable, + }, + else => {}, + } + } + + fn lowerReg(s: *const Select, reg: Register) Register { + return if (reg.isClass(.x87)) @enumFromInt(@intFromEnum(Register.st0) + (@as(u3, @intCast(reg.enc())) -% s.top)) else reg; + } + + const Case = struct { + required_abi: enum { any, gnu, msvc } = .any, + required_cc_abi: enum { any, sysv64, win64 } = .any, + required_features: [4]?std.Target.x86.Feature = @splat(null), + src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any), + dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any), + patterns: []const Select.Pattern, + call_frame: packed struct(u16) { size: u10 = 0, alignment: InternPool.Alignment } = .{ .size = 0, .alignment = .none }, + extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused), + dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused), + clobbers: packed struct { + eflags: bool = false, + caller_preserved: CallConv = .none, + } = .{}, + each: union(enum) { + once: []const Instruction, + }, + + const CallConv = enum(u2) { none, ccc, zigcc }; + }; + + const Constraint = union(enum) { + any, + any_bool_vec, + any_int, + any_signed_int, + any_unsigned_int, + any_scalar_int, + any_scalar_signed_int, + any_scalar_unsigned_int, + any_float, + po2_any, + bool, + bool_vec: Memory.Size, + exact_bool_vec: u16, + ptr_any_bool_vec, + ptr_bool_vec: Memory.Size, + ptr_any_bool_vec_elem, + ptr_bool_vec_elem: Memory.Size, + remainder_bool_vec: OfIsSizes, + exact_remainder_bool_vec: struct { of: Memory.Size, is: u16 }, + signed_int_vec: Memory.Size, + signed_int_or_full_vec: Memory.Size, + unsigned_int_vec: Memory.Size, + size: Memory.Size, + multiple_size: Memory.Size, + int: Memory.Size, + int_or_float: Memory.Size, + scalar_int_is: Memory.Size, + scalar_signed_int_is: Memory.Size, + scalar_unsigned_int_is: Memory.Size, + scalar: OfIsSizes, + scalar_int: OfIsSizes, + scalar_signed_int: OfIsSizes, + scalar_unsigned_int: OfIsSizes, + exclusive_scalar_int: OfIsSizes, + exclusive_scalar_signed_int: OfIsSizes, + exclusive_scalar_unsigned_int: OfIsSizes, + exact_scalar_int: OfIsSizes, + exact_scalar_signed_int: OfIsSizes, + exact_scalar_unsigned_int: OfIsSizes, + scalar_signed_or_exclusive_int: OfIsSizes, + scalar_exact_int: struct { of: Memory.Size, is: u16 }, + scalar_exact_signed_int: struct { of: Memory.Size, is: u16 }, + scalar_exact_unsigned_int: struct { of: Memory.Size, is: u16 }, + multiple_scalar: OfIsSizes, + multiple_scalar_int: OfIsSizes, + unaligned_multiple_scalar_int: OfIsSizes, + unaligned_multiple_scalar_signed_int: OfIsSizes, + unaligned_multiple_scalar_unsigned_int: OfIsSizes, + multiple_scalar_signed_int: OfIsSizes, + multiple_scalar_unsigned_int: OfIsSizes, + multiple_scalar_signed_or_exclusive_int: OfIsSizes, + multiple_scalar_exact_int: struct { of: Memory.Size, is: u16 }, + multiple_scalar_exact_signed_int: struct { of: Memory.Size, is: u16 }, + multiple_scalar_exact_unsigned_int: struct { of: Memory.Size, is: u16 }, + scalar_remainder_int: OfIsSizes, + scalar_remainder_signed_int: OfIsSizes, + scalar_remainder_unsigned_int: OfIsSizes, + scalar_exact_remainder_int: OfIsSizes, + scalar_exact_remainder_signed_int: OfIsSizes, + scalar_exact_remainder_unsigned_int: OfIsSizes, + float: Memory.Size, + scalar_any_float: Memory.Size, + scalar_float: OfIsSizes, + exclusive_scalar_float: OfIsSizes, + exact_scalar_float: OfIsSizes, + multiple_scalar_any_float: Memory.Size, + multiple_scalar_float: OfIsSizes, + unaligned_multiple_scalar_float: OfIsSizes, + exact_int: u16, + exact_signed_int: u16, + exact_unsigned_int: u16, + signed_or_exact_int: Memory.Size, + unsigned_or_exact_int: Memory.Size, + signed_or_exclusive_int: Memory.Size, + po2_int: Memory.Size, + signed_po2_int: Memory.Size, + unsigned_po2_or_exact_int: Memory.Size, + remainder_int: OfIsSizes, + remainder_signed_int: OfIsSizes, + remainder_unsigned_int: OfIsSizes, + exact_remainder_int: struct { of: Memory.Size, is: u16 }, + exact_remainder_signed_int: struct { of: Memory.Size, is: u16 }, + exact_remainder_unsigned_int: struct { of: Memory.Size, is: u16 }, + signed_or_exact_remainder_int: OfIsSizes, + unsigned_or_exact_remainder_int: OfIsSizes, + signed_int: Memory.Size, + unsigned_int: Memory.Size, + elem_size_is: u8, + po2_elem_size, + elem_int: Memory.Size, + + const OfIsSizes = struct { of: Memory.Size, is: Memory.Size }; + + fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool { + const zcu = cg.pt.zcu; + return switch (constraint) { + .any => true, + .any_bool_vec => ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type, + .any_int => cg.intInfo(ty) != null, + .any_signed_int => if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed else false, + .any_unsigned_int => if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned else false, + .any_scalar_int => cg.intInfo(ty.scalarType(zcu)) != null, + .any_scalar_signed_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed else false, + .any_scalar_unsigned_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned else false, + .any_float => ty.isRuntimeFloat(), + .po2_any => std.math.isPowerOfTwo(ty.abiSize(zcu)), + .bool => ty.toIntern() == .bool_type, + .bool_vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and + size.bitSize(cg.target) >= ty.vectorLen(zcu), + .exact_bool_vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and + size == ty.vectorLen(zcu), + .ptr_any_bool_vec => switch (zcu.intern_pool.indexToKey(ty.childType(zcu).toIntern())) { + .vector_type => |vector_type| vector_type.child == .bool_type, + else => false, + }, + .ptr_bool_vec => |size| switch (zcu.intern_pool.indexToKey(ty.childType(zcu).toIntern())) { + .vector_type => |vector_type| vector_type.child == .bool_type and size.bitSize(cg.target) >= vector_type.len, + else => false, + }, + .ptr_any_bool_vec_elem => { + const ptr_info = ty.ptrInfo(zcu); + return switch (ptr_info.flags.vector_index) { + .none => false, + .runtime => unreachable, + else => ptr_info.child == .bool_type, + }; + }, + .ptr_bool_vec_elem => |size| { + const ptr_info = ty.ptrInfo(zcu); + return switch (ptr_info.flags.vector_index) { + .none => false, + .runtime => unreachable, + else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size, + }; + }, + .remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and + of_is.is.bitSize(cg.target) >= (ty.vectorLen(zcu) - 1) % of_is.of.bitSize(cg.target) + 1, + .exact_remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and + of_is.is == (ty.vectorLen(zcu) - 1) % of_is.of.bitSize(cg.target) + 1, + .signed_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .signed else false, + .signed_int_or_full_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + if (cg.intInfo(ty.childType(zcu))) |int_info| switch (int_info.signedness) { + .signed => true, + .unsigned => int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits), + } else false, + .unsigned_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .unsigned else false, + .size => |size| @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu), + .multiple_size => |size| ty.abiSize(zcu) % @divExact(size.bitSize(cg.target), 8) == 0, + .int => |size| if (cg.intInfo(ty)) |int_info| size.bitSize(cg.target) >= int_info.bits else false, + .int_or_float => |size| if (cg.intInfo(ty)) |int_info| + size.bitSize(cg.target) >= int_info.bits + else if (cg.floatBits(ty)) |float_bits| + size.bitSize(cg.target) == float_bits + else + false, + .scalar_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| + size.bitSize(cg.target) >= int_info.bits + else + false, + .scalar_signed_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { + .signed => size.bitSize(cg.target) >= int_info.bits, + .unsigned => false, + } else false, + .scalar_unsigned_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { + .signed => false, + .unsigned => size.bitSize(cg.target) >= int_info.bits, + } else false, + .scalar => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| + of_is.is.bitSize(cg.target) >= int_info.bits + else if (cg.floatBits(ty.scalarType(zcu))) |float_bits| + of_is.is.bitSize(cg.target) == float_bits + else + false, + .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .exclusive_scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .exclusive_scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .exclusive_scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .exact_scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .exact_scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .exact_scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .scalar_signed_or_exclusive_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => of_is.is.bitSize(cg.target) >= int_info.bits, + .unsigned => of_is.is.bitSize(cg.target) > int_info.bits, + } else false, + .scalar_exact_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is == int_info.bits else false, + .scalar_exact_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is == int_info.bits else false, + .scalar_exact_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is == int_info.bits else false, + .multiple_scalar => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| + of_is.is.bitSize(cg.target) >= int_info.bits + else if (cg.floatBits(ty.scalarType(zcu))) |float_bits| + of_is.is.bitSize(cg.target) == float_bits + else + false, + .multiple_scalar_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .unaligned_multiple_scalar_int => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .unaligned_multiple_scalar_signed_int => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .unaligned_multiple_scalar_unsigned_int => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .multiple_scalar_signed_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .multiple_scalar_unsigned_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .multiple_scalar_signed_or_exclusive_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { + .signed => of_is.is.bitSize(cg.target) >= int_info.bits, + .unsigned => of_is.is.bitSize(cg.target) > int_info.bits, + } else false, + .multiple_scalar_exact_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is == int_info.bits else false, + .multiple_scalar_exact_signed_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is == int_info.bits else false, + .multiple_scalar_exact_unsigned_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is == int_info.bits else false, + .scalar_remainder_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| + of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 + else + false, + .scalar_remainder_signed_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .scalar_remainder_unsigned_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .scalar_exact_remainder_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| + of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 + else + false, + .scalar_exact_remainder_signed_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .scalar_exact_remainder_unsigned_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .float => |size| if (cg.floatBits(ty)) |float_bits| size.bitSize(cg.target) == float_bits else false, + .scalar_any_float => |size| @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + cg.floatBits(ty.scalarType(zcu)) != null, + .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, + .exclusive_scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) > cg.unalignedSize(ty) and + if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, + .exact_scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) == cg.unalignedSize(ty) and + if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, + .multiple_scalar_any_float => |size| ty.abiSize(zcu) % @divExact(size.bitSize(cg.target), 8) == 0 and + cg.floatBits(ty.scalarType(zcu)) != null, + .multiple_scalar_float => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, + .unaligned_multiple_scalar_float => |of_is| cg.unalignedSize(ty) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, + .exact_int => |bit_size| if (cg.intInfo(ty)) |int_info| bit_size == int_info.bits else false, + .exact_signed_int => |bit_size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => bit_size == int_info.bits, + .unsigned => false, + } else false, + .exact_unsigned_int => |bit_size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => false, + .unsigned => bit_size == int_info.bits, + } else false, + .signed_or_exact_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => size.bitSize(cg.target) >= int_info.bits, + .unsigned => size.bitSize(cg.target) == int_info.bits, + } else false, + .unsigned_or_exact_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => size.bitSize(cg.target) == int_info.bits, + .unsigned => size.bitSize(cg.target) >= int_info.bits, + } else false, + .signed_or_exclusive_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => size.bitSize(cg.target) >= int_info.bits, + .unsigned => size.bitSize(cg.target) > int_info.bits, + } else false, + .po2_int => |size| if (cg.intInfo(ty)) |int_info| + std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits + else + false, + .signed_po2_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits, + .unsigned => false, + } else false, + .unsigned_po2_or_exact_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => size.bitSize(cg.target) == int_info.bits, + .unsigned => std.math.isPowerOfTwo(int_info.bits) and size.bitSize(cg.target) >= int_info.bits, + } else false, + .remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| + of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 + else + false, + .remainder_signed_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .remainder_unsigned_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| + of_is.is == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 + else + false, + .exact_remainder_signed_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed and + of_is.is == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .exact_remainder_unsigned_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned and + of_is.is == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .signed_or_exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, + .unsigned => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, + } else false, + .unsigned_or_exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, + .unsigned => of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1, + } else false, + .signed_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => size.bitSize(cg.target) >= int_info.bits, + .unsigned => false, + } else false, + .unsigned_int => |size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { + .signed => false, + .unsigned => size.bitSize(cg.target) >= int_info.bits, + } else false, + .elem_size_is => |size| size == ty.elemType2(zcu).abiSize(zcu), + .po2_elem_size => std.math.isPowerOfTwo(ty.elemType2(zcu).abiSize(zcu)), + .elem_int => |size| if (cg.intInfo(ty.elemType2(zcu))) |elem_int_info| + size.bitSize(cg.target) >= elem_int_info.bits + else + false, + }; + } + }; + + const Pattern = struct { + src: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Src, + commute: struct { u8, u8 } = .{ 0, 0 }, + + const Src = union(enum) { + none, + any, + imm: i16, + imm8, + imm16, + imm32, + simm32, + to_reg: Register, + to_reg_pair: [2]Register, + to_param_gpr: TempSpec.Kind.CallConvRegSpec, + to_param_gpr_pair: TempSpec.Kind.CallConvRegSpec, + to_param_sse: TempSpec.Kind.CallConvRegSpec, + to_ret_gpr: TempSpec.Kind.CallConvRegSpec, + to_ret_gpr_pair: TempSpec.Kind.CallConvRegSpec, + mem, + to_mem, + mut_mem, + to_mut_mem, + gpr, + immut_gpr, + to_gpr, + mut_gpr, + to_mut_gpr, + to_gphi, + to_mut_gphi, + x87, + to_x87, + mut_x87, + to_mut_x87, + mmx, + to_mmx, + mut_mmx, + to_mut_mmx, + mm, + to_mm, + mut_mm, + to_mut_mm, + sse, + to_sse, + mut_sse, + to_mut_sse, + xmm, + to_xmm, + mut_xmm, + to_mut_xmm, + ymm, + to_ymm, + mut_ymm, + to_mut_ymm, + reg_mask: RegMaskSpec, + all_reg_mask: RegMaskSpec, + + const RegMaskSpec = struct { + size: Memory.Size, + is: enum { + any, + uninverted, + inverted, + + fn matches(is: @This(), inverted: bool) bool { + return switch (is) { + .any => true, + .uninverted => !inverted, + .inverted => inverted, + }; + } + } = .any, + }; + + fn matches(src: Src, temp: Temp, cg: *CodeGen) bool { + return switch (src) { + .none => temp.tracking(cg).short == .none, + .any => true, + .imm => |specific_imm| switch (temp.tracking(cg).short) { + .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm, + else => false, + }, + .imm8 => switch (temp.tracking(cg).short) { + .immediate => |imm| std.math.cast(u8, imm) != null or + std.math.cast(i8, @as(i64, @bitCast(imm))) != null, + else => false, + }, + .imm16 => switch (temp.tracking(cg).short) { + .immediate => |imm| std.math.cast(u16, imm) != null or + std.math.cast(i16, @as(i64, @bitCast(imm))) != null, + else => false, + }, + .imm32 => switch (temp.tracking(cg).short) { + .immediate => |imm| std.math.cast(u32, imm) != null or + std.math.cast(i32, @as(i64, @bitCast(imm))) != null, + else => false, + }, + .simm32 => switch (temp.tracking(cg).short) { + .immediate => |imm| std.math.cast(i32, @as(i64, @bitCast(imm))) != null, + else => false, + }, + .mem => temp.tracking(cg).short.isMemory(), + .to_mem, .to_mut_mem => true, + .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(), + .to_reg, .to_reg_pair, .to_param_gpr, .to_param_gpr_pair, .to_param_sse, .to_ret_gpr, .to_ret_gpr_pair => true, + .gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.general_purpose), + .register_offset => |reg_off| reg_off.reg.isClass(.general_purpose) and reg_off.off == 0, + else => false, + }, + .immut_gpr => !temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.general_purpose), + .register_offset => |reg_off| reg_off.reg.isClass(.general_purpose) and reg_off.off == 0, + else => false, + }, + .mut_gpr => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.general_purpose), + .register_offset => |reg_off| reg_off.reg.isClass(.general_purpose) and reg_off.off == 0, + else => false, + }, + .to_gpr, .to_mut_gpr, .to_gphi, .to_mut_gphi => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8, + .x87 => switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.x87), + .register_offset => |reg_off| reg_off.reg.isClass(.x87) and reg_off.off == 0, + else => false, + }, + .mut_x87 => temp.isMut(cg) and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.x87), + .register_offset => |reg_off| reg_off.reg.isClass(.x87) and reg_off.off == 0, + else => false, + }, + .to_x87, .to_mut_x87 => true, + .mmx => switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.mmx), + .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, + else => false, + }, + .mut_mmx => temp.isMut(cg) and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.mmx), + .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, + else => false, + }, + .to_mmx, .to_mut_mmx => true, + .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.mmx), + .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, + else => false, + }, + .mut_mm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.mmx), + .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0, + else => false, + }, + .to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8, + .sse => switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.sse), + .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, + else => false, + }, + .mut_sse => temp.isMut(cg) and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.sse), + .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, + else => false, + }, + .to_sse, .to_mut_sse => true, + .xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.sse), + .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, + else => false, + }, + .mut_xmm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.sse), + .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, + else => false, + }, + .to_xmm, .to_mut_xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16, + .ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.sse), + .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, + else => false, + }, + .mut_ymm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) { + .register => |reg| reg.isClass(.sse), + .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0, + else => false, + }, + .to_ymm, .to_mut_ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32, + .reg_mask => |mask_spec| switch (temp.tracking(cg).short) { + .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) >= + reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and + mask_spec.is.matches(reg_mask.info.inverted), + else => false, + }, + .all_reg_mask => |mask_spec| switch (temp.tracking(cg).short) { + .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) == + reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and + reg_mask.info.kind == .all and mask_spec.is.matches(reg_mask.info.inverted), + else => false, + }, + }; + } + + fn convert(src: Src, temp: *Temp, cg: *CodeGen) InnerError!bool { + return switch (src) { + .none, .any, .imm, .imm8, .imm16, .imm32, .simm32, .reg_mask, .all_reg_mask => false, + .mem, .to_mem => try temp.toBase(false, cg), + .mut_mem, .to_mut_mem => try temp.toBase(true, cg), + .to_reg => |reg| try temp.toReg(reg, cg), + .to_reg_pair => |regs| try temp.toRegPair(regs, cg), + .to_param_gpr => |param_spec| try temp.toReg(abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)], cg), + .to_param_gpr_pair => |param_spec| try temp.toRegPair(abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..2].*, cg), + .to_param_sse => |param_spec| try temp.toReg(abi.getCAbiSseParamRegs(param_spec.tag(cg), cg.target)[param_spec.index(cg)], cg), + .to_ret_gpr => |ret_spec| try temp.toReg(abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)], cg), + .to_ret_gpr_pair => |ret_spec| try temp.toRegPair(abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..2].*, cg), + .gpr, .immut_gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg), + .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg), + .to_gphi => try temp.toRegClass(false, .gphi, cg), + .to_mut_gphi => try temp.toRegClass(true, .gphi, cg), + .x87, .to_x87 => try temp.toRegClass(false, .x87, cg), + .mut_x87, .to_mut_x87 => try temp.toRegClass(true, .x87, cg), + .mmx, .to_mmx, .mm, .to_mm => try temp.toRegClass(false, .mmx, cg), + .mut_mmx, .to_mut_mmx, .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg), + .sse, .to_sse, .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg), + .mut_sse, .to_mut_sse, .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg), + }; + } + }; + }; + + const TempSpec = struct { + type: Type = .noreturn, + kind: Kind, + + const unused: TempSpec = .{ .kind = .unused }; + + const Kind = union(enum) { + unused, + any, + none, + undef, + cc: Condition, + ptest_cc: struct { ref: Select.Operand.Ref, not: bool = false, all: u1 }, + imm: i32, + ref: Select.Operand.Ref, + reg: Register, + reg_pair: [2]Register, + param_gpr: CallConvRegSpec, + param_gpr_pair: CallConvRegSpec, + param_sse: CallConvRegSpec, + ret_gpr: CallConvRegSpec, + ret_gpr_pair: CallConvRegSpec, + ret_sse: CallConvRegSpec, + rc: Register.Class, + rc_pair: Register.Class, + mut_rc: struct { ref: Select.Operand.Ref, rc: Register.Class }, + ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, + rc_mask: struct { rc: Register.Class, info: MaskInfo }, + mut_rc_mask: struct { ref: Select.Operand.Ref, rc: Register.Class, info: MaskInfo }, + mem, + mem_of_type: Select.Operand.Ref, + smin_mem: ConstSpec, + smax_mem: ConstSpec, + slimit_delta_mem: ConstSpec, + umin_mem: ConstSpec, + umax_mem: ConstSpec, + umax_delta_mem: ConstSpec, + @"0x1p63_mem": ConstSpec, + f64_0x1p52_0x1p84_mem, + u32_0x1p52_hi_0x1p84_hi_0_0_mem, + f32_0_0x1p64_mem, + pshufb_splat_mem: struct { of: Memory.Size = .none, size: Memory.Size }, + pshufb_trunc_mem: struct { of: Memory.Size = .none, from: Memory.Size, to: Memory.Size }, + pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size }, + pand_mask_mem: struct { ref: Select.Operand.Ref, invert: bool = false }, + ptest_mask_mem: Select.Operand.Ref, + pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 }, + bits_mem: enum { forward, reverse }, + splat_int_mem: struct { ref: Select.Operand.Ref, inside: enum { umin, smin, smax } = .umin, outside: enum { smin, smax } }, + splat_float_mem: struct { ref: Select.Operand.Ref, inside: enum { zero } = .zero, outside: f16 }, + frame: FrameIndex, + lazy_sym: struct { kind: link.File.LazySymbol.Kind, ref: Select.Operand.Ref = .none }, + extern_func: [*:0]const u8, + + const ConstSpec = struct { + ref: Select.Operand.Ref = .none, + to_signedness: ?std.builtin.Signedness = null, + vectorize_to: ?Memory.Size = null, + }; + + const CallConvRegSpec = struct { + cc: Case.CallConv, + after: u2, + at: u2, + + fn tag(spec: CallConvRegSpec, cg: *const CodeGen) std.builtin.CallingConvention.Tag { + return switch (spec.cc) { + .none => unreachable, + .ccc => cg.target.cCallingConvention().?, + .zigcc => .auto, + }; + } + + fn index(spec: CallConvRegSpec, cg: *const CodeGen) u2 { + return switch (spec.tag(cg)) { + .auto, .x86_64_sysv => spec.after, + .x86_64_win => spec.at, + else => unreachable, + }; + } + }; + + fn lock(kind: Kind, cg: *CodeGen) ![2]?RegisterLock { + var reg_locks: [2]?RegisterLock = @splat(null); + const regs: [2]Register = switch (kind) { + else => return reg_locks, + .reg => |reg| .{ reg, .none }, + .reg_pair => |regs| regs, + .param_gpr => |param_spec| abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..1].* ++ .{.none}, + .param_gpr_pair => |param_spec| abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..2].*, + .param_sse => |param_spec| abi.getCAbiSseParamRegs(param_spec.tag(cg), cg.target)[param_spec.index(cg)..][0..1].* ++ .{.none}, + .ret_gpr => |ret_spec| abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..1].* ++ .{.none}, + .ret_gpr_pair => |ret_spec| abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..2].*, + .ret_sse => |ret_spec| abi.getCAbiSseReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..1].* ++ .{.none}, + }; + for (regs, ®_locks) |reg, *reg_lock| { + if (reg == .none) continue; + const reg_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue; + try cg.register_manager.getRegIndex(reg_index, null); + reg_lock.* = cg.register_manager.lockRegIndex(reg_index); + } + return reg_locks; + } + }; + + fn create(spec: TempSpec, s: *const Select) InnerError!struct { Temp, bool } { + const cg = s.cg; + const pt = cg.pt; + return switch (spec.kind) { + .unused => .{ undefined, false }, + .any => .{ try cg.tempAlloc(spec.type), true }, + .none => .{ try cg.tempInit(spec.type, .none), true }, + .undef => .{ try cg.tempInit(spec.type, .undef), true }, + .imm => |imm| .{ try cg.tempInit(spec.type, .{ .immediate = @bitCast(@as(i64, imm)) }), true }, + .cc => |cc| .{ try cg.tempInit(spec.type, .{ .eflags = cc }), true }, + .ptest_cc => |cc_spec| { + const ccs: [2]Condition = .{ .z, .c }; + const cc = ccs[cc_spec.all ^ @intFromBool(cc_spec.ref.valueOf(s).register_mask.info.inverted)]; + return .{ try cg.tempInit(spec.type, .{ .eflags = switch (cc_spec.not) { + false => cc, + true => cc.negate(), + } }), true }; + }, + .ref => |ref| .{ ref.tempOf(s), false }, + .reg => |reg| .{ try cg.tempInit(spec.type, .{ .register = reg }), true }, + .reg_pair => |regs| .{ try cg.tempInit(spec.type, .{ .register_pair = regs }), true }, + .param_gpr => |param_spec| .{ try cg.tempInit(spec.type, .{ + .register = abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)], + }), true }, + .param_gpr_pair => |param_spec| .{ try cg.tempInit(spec.type, .{ + .register_pair = abi.getCAbiIntParamRegs(param_spec.tag(cg))[param_spec.index(cg)..][0..2].*, + }), true }, + .param_sse => |param_spec| .{ try cg.tempInit(spec.type, .{ + .register = abi.getCAbiSseParamRegs(param_spec.tag(cg), cg.target)[param_spec.index(cg)], + }), true }, + .ret_gpr => |ret_spec| .{ try cg.tempInit(spec.type, .{ + .register = abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)], + }), true }, + .ret_gpr_pair => |ret_spec| .{ try cg.tempInit(spec.type, .{ + .register_pair = abi.getCAbiIntReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)..][0..2].*, + }), true }, + .ret_sse => |ret_spec| .{ try cg.tempInit(spec.type, .{ + .register = abi.getCAbiSseReturnRegs(ret_spec.tag(cg))[ret_spec.index(cg)], + }), true }, + .rc => |rc| .{ try cg.tempAllocReg(spec.type, regSetForRegClass(rc)), true }, + .rc_pair => |rc| .{ try cg.tempAllocRegPair(spec.type, regSetForRegClass(rc)), true }, + .mut_rc => |ref_rc| { + const temp = ref_rc.ref.tempOf(s); + if (temp.isMut(cg)) switch (temp.tracking(cg).short) { + .register => |reg| if (reg.isClass(ref_rc.rc)) return .{ temp, false }, + .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.isClass(ref_rc.rc)) return .{ temp, false }, + else => {}, + }; + return .{ try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc)), true }; + }, + .ref_mask => |ref_mask| .{ ref_mask.ref.tempOf(s), false }, + .rc_mask => |rc_mask| .{ try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)), true }, + .mut_rc_mask => |ref_rc_mask| { + const temp = ref_rc_mask.ref.tempOf(s); + if (temp.isMut(cg)) switch (temp.tracking(cg).short) { + .register => |reg| if (reg.isClass(ref_rc_mask.rc)) return .{ temp, false }, + .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.isClass(ref_rc_mask.rc)) return .{ temp, false }, + else => {}, + }; + return .{ try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc_mask.rc)), true }; + }, + .mem => .{ try cg.tempAllocMem(spec.type), true }, + .mem_of_type => |ref| .{ try cg.tempAllocMem(ref.typeOf(s)), true }, + .smin_mem, + .smax_mem, + .slimit_delta_mem, + .umin_mem, + .umax_mem, + .umax_delta_mem, + .@"0x1p63_mem", + => |const_spec| { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty = if (const_spec.ref == .none) spec.type else const_spec.ref.typeOf(s); + const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) { + else => .{ null, ty }, + .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) }, + }; + const res_vector_len: ?u32 = if (const_spec.vectorize_to) |vectorize_to| switch (vectorize_to) { + .none => null, + else => @intCast(@divExact(@divExact(vectorize_to.bitSize(cg.target), 8), scalar_ty.abiSize(pt.zcu))), + } else vector_len; + const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) { + .bool_type => .{ scalar_ty, .fromInterned(switch (spec.kind) { + else => unreachable, + .smin_mem, .umax_mem => .bool_true, + .smax_mem, .umin_mem => .bool_false, + }) }, + else => { + const scalar_info: std.builtin.Type.Int = cg.intInfo(scalar_ty) orelse .{ + .signedness = .signed, + .bits = cg.floatBits(scalar_ty).?, + }; + const res_scalar_info: std.builtin.Type.Int = .{ + .signedness = const_spec.to_signedness orelse scalar_info.signedness, + .bits = switch (spec.kind) { + else => scalar_info.bits, + .slimit_delta_mem, .umax_delta_mem => cg.intInfo(spec.type.scalarType(zcu)).?.bits, + }, + }; + const res_scalar_ty = try pt.intType(res_scalar_info.signedness, res_scalar_info.bits); + + if (res_scalar_info.bits <= 64) { + const int_val: i64 = switch (spec.kind) { + else => unreachable, + .smin_mem => std.math.minInt(i64), + .smax_mem => std.math.maxInt(i64), + .slimit_delta_mem => std.math.maxInt(i64) - + (@as(i64, std.math.maxInt(i64)) >> @intCast(res_scalar_info.bits - scalar_info.bits)), + .umin_mem => 0, + .umax_mem => -1, + .umax_delta_mem => @bitCast(std.math.maxInt(u64) - + (@as(u64, std.math.maxInt(u64)) >> @intCast(res_scalar_info.bits - scalar_info.bits))), + .@"0x1p63_mem" => switch (res_scalar_info.bits) { + else => unreachable, + 16 => @as(i64, @as(i16, @bitCast(@as(f16, 0x1p63)))) << 64 - 16, + 32 => @as(i64, @as(i32, @bitCast(@as(f32, 0x1p63)))) << 64 - 32, + 64 => @as(i64, @as(i64, @bitCast(@as(f64, 0x1p63)))) << 64 - 64, + }, + }; + const shift: u6 = @intCast(64 - res_scalar_info.bits); + break :res_scalar .{ res_scalar_ty, switch (res_scalar_info.signedness) { + .signed => try pt.intValue_i64(res_scalar_ty, int_val >> shift), + .unsigned => try pt.intValue_u64(res_scalar_ty, @as(u64, @bitCast(int_val)) >> shift), + } }; + } + + const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(1 << 10)]std.math.big.Limb; + var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = + std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); + const allocator = stack.get(); + var res_big_int: std.math.big.int.Mutable = .{ + .limbs = try allocator.alloc( + std.math.big.Limb, + std.math.big.int.calcTwosCompLimbCount(res_scalar_info.bits), + ), + .len = undefined, + .positive = undefined, + }; + defer allocator.free(res_big_int.limbs); + switch (spec.kind) { + else => unreachable, + .smin_mem, .smax_mem, .umin_mem, .umax_mem => res_big_int.setTwosCompIntLimit(switch (spec.kind) { + else => unreachable, + .smin_mem, .umin_mem => .min, + .smax_mem, .umax_mem => .max, + }, switch (spec.kind) { + else => unreachable, + .smin_mem, .smax_mem => .signed, + .umin_mem, .umax_mem => .unsigned, + }, res_scalar_info.bits), + .slimit_delta_mem, .umax_delta_mem => { + var big_int: std.math.big.int.Mutable = .{ + .limbs = try allocator.alloc( + std.math.big.Limb, + std.math.big.int.calcTwosCompLimbCount(scalar_info.bits), + ), + .len = undefined, + .positive = undefined, + }; + defer allocator.free(big_int.limbs); + const signedness: std.builtin.Signedness = switch (spec.kind) { + else => unreachable, + .slimit_delta_mem => .signed, + .umax_delta_mem => .unsigned, + }; + big_int.setTwosCompIntLimit(.max, signedness, scalar_info.bits); + res_big_int.setTwosCompIntLimit(.max, signedness, res_scalar_info.bits); + res_big_int.sub(res_big_int.toConst(), big_int.toConst()); + }, + .@"0x1p63_mem" => switch (res_scalar_info.bits) { + else => unreachable, + 80 => res_big_int.set(@as(u80, @bitCast(@as(f80, 0x1p63)))), + 128 => res_big_int.set(@as(u128, @bitCast(@as(f128, 0x1p63)))), + }, + } + res_big_int.truncate(res_big_int.toConst(), res_scalar_info.signedness, res_scalar_info.bits); + break :res_scalar .{ res_scalar_ty, try pt.intValue_big(res_scalar_ty, res_big_int.toConst()) }; + }, + }; + const res_val = if (res_vector_len) |len| try pt.aggregateSplatValue(try pt.vectorType(.{ + .len = len, + .child = res_scalar_ty.toIntern(), + }), res_scalar_val) else res_scalar_val; + return .{ try cg.tempMemFromValue(res_val), true }; + }, + .f64_0x1p52_0x1p84_mem => .{ try cg.tempMemFromValue( + try pt.aggregateValue(try pt.vectorType(.{ .len = 2, .child = .f64_type }), &.{ + (try pt.floatValue(.f64, @as(f64, 0x1p52))).toIntern(), + (try pt.floatValue(.f64, @as(f64, 0x1p84))).toIntern(), + }), + ), true }, + .u32_0x1p52_hi_0x1p84_hi_0_0_mem => .{ try cg.tempMemFromValue( + try pt.aggregateValue(try pt.vectorType(.{ .len = 4, .child = .u32_type }), &(.{ + (try pt.intValue(.u32, @as(u64, @bitCast(@as(f64, 0x1p52))) >> 32)).toIntern(), + (try pt.intValue(.u32, @as(u64, @bitCast(@as(f64, 0x1p84))) >> 32)).toIntern(), + } ++ .{(try pt.intValue(.u32, 0)).toIntern()} ** 2)), + ), true }, + .f32_0_0x1p64_mem => .{ try cg.tempMemFromValue( + try pt.aggregateValue(try pt.vectorType(.{ .len = 2, .child = .f32_type }), &.{ + (try pt.floatValue(.f32, @as(f32, 0))).toIntern(), + (try pt.floatValue(.f32, @as(f32, 0x1p64))).toIntern(), + }), + ), true }, + .pshufb_splat_mem => |splat_spec| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + var elem_buf: [32]u8 = @splat(1 << 7); + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const of_bytes: u6 = @intCast(switch (splat_spec.of) { + .none => elems.len, + else => |of| @divExact(of.bitSize(cg.target), 8), + }); + const size_bytes: u6 = @intCast(@divExact(splat_spec.size.bitSize(cg.target), 8)); + var index: u6 = 0; + while (index < of_bytes) : (index += size_bytes) for (0..size_bytes) |byte_off| { + elems[index + byte_off] = @as(u4, @truncate(byte_off)); + }; + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .pshufb_trunc_mem => |trunc_spec| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + var elem_buf: [32]u8 = @splat(1 << 7); + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const of_bytes: u6 = @intCast(switch (trunc_spec.of) { + .none => elems.len, + else => |of| @divExact(of.bitSize(cg.target), 8), + }); + const from_bytes: u5 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8)); + const to_bytes: u5 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8)); + var from_index: u6 = 0; + var to_index: u6 = 0; + while (from_index < of_bytes) : ({ + from_index += from_bytes; + switch (from_index % 16) { + 0 => to_index = from_index, + else => to_index += to_bytes, + } + }) for (0..to_bytes) |byte_off| { + elems[to_index + byte_off] = @as(u4, @truncate(from_index + byte_off)); + }; + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .pand_trunc_mem => |trunc_spec| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + var elem_buf: [32]u8 = @splat(0); + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const from_bytes: u6 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8)); + const to_bytes: u6 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8)); + var index: u6 = 0; + while (index < elems.len) : (index += from_bytes) @memset(elems[index..][0..to_bytes], std.math.maxInt(u8)); + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .pand_mask_mem => |mask_spec| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + const ref_ty = mask_spec.ref.typeOf(s); + assert(ref_ty.isVector(zcu)); + var elem_buf: [64]u8 = undefined; + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const mask_len: usize = @intCast((cg.unalignedSize(ref_ty) - 1) % elems.len + 1); + const invert_mask: u8 = switch (mask_spec.invert) { + false => std.math.minInt(u8), + true => std.math.maxInt(u8), + }; + @memset(elems[0..mask_len], ~invert_mask); + @memset(elems[mask_len..], invert_mask); + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .ptest_mask_mem => |mask_ref| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + const ref_ty = mask_ref.typeOf(s); + assert(ref_ty.isVector(zcu) and ref_ty.childType(zcu).toIntern() == .bool_type); + const mask_info = mask_ref.valueOf(s).register_mask.info; + var elem_buf: [64]u8 = @splat(0); + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const elem_bytes: u6 = @intCast(@divExact(mask_info.scalar.bitSize(cg.target), 8)); + var index: u7 = 0; + for (0..@intCast(ref_ty.vectorLen(zcu))) |_| { + switch (mask_info.kind) { + .sign => { + @memset(elems[index..][0 .. elem_bytes - 1], std.math.minInt(u8)); + elems[index + elem_bytes - 1] = @bitCast(@as(i8, std.math.minInt(i8))); + }, + .all => @memset(elems[index..][0..elem_bytes], std.math.maxInt(u8)), + } + index += elem_bytes; + } + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .pshufb_bswap_mem => |bswap_spec| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + var elem_buf: [32]u8 = @splat(1 << 7); + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8)); + var to_index: u6 = 0; + for (0..bswap_spec.repeat) |_| for (0..len) |from_index| { + @memset(elems[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index)); + to_index += bswap_spec.smear; + }; + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .bits_mem => |direction| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type); + var bytes: [32]u8 = undefined; + const elems = bytes[0..spec.type.vectorLen(zcu)]; + for (elems, 0..) |*elem, index| elem.* = switch (direction) { + .forward => @as(u8, 1 << 0) << @truncate(index), + .reverse => @as(u8, 1 << 7) >> @truncate(index), + }; + return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{ + .ty = spec.type.toIntern(), + .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) }, + } }))), true }; + }, + .splat_int_mem => |splat_spec| { + const zcu = pt.zcu; + const elem_ty = spec.type.scalarType(zcu); + const elem_bits = cg.intInfo(elem_ty).?.bits; + const ref_ty = splat_spec.ref.typeOf(s); + assert(ref_ty.isVector(zcu) and cg.intInfo(ref_ty.childType(zcu)).?.bits <= elem_bits); + var elem_buf: [64]InternPool.Index = undefined; + const elems = elem_buf[0..if (elem_ty.toIntern() == spec.type.toIntern()) 1 else spec.type.vectorLen(zcu)]; + const inside_len = (ref_ty.vectorLen(zcu) - 1) % elems.len + 1; + @memset(elems[0..inside_len], (try pt.intValue(elem_ty, @as(i64, switch (splat_spec.inside) { + .umin => 0, + .smin => std.math.minInt(i64), + .smax => std.math.maxInt(i64), + }) >> @intCast(64 - elem_bits))).toIntern()); + @memset(elems[inside_len..], (try pt.intValue(elem_ty, @as(i64, switch (splat_spec.outside) { + .smin => std.math.minInt(i64), + .smax => std.math.maxInt(i64), + }) >> @intCast(64 - elem_bits))).toIntern()); + if (elems.len == 1) return .{ try cg.tempMemFromValue(.fromInterned(elems[0])), true }; + const mem_size = cg.unalignedSize(spec.type); + return .{ try cg.tempMemFromAlignedValue( + if (mem_size < 16) .fromByteUnits(mem_size) else .none, + try pt.aggregateValue(if (mem_size < 16) try pt.arrayType(.{ + .len = elems.len, + .child = elem_ty.toIntern(), + }) else spec.type, elems), + ), true }; + }, + .splat_float_mem => |splat_spec| { + const zcu = pt.zcu; + assert(spec.type.isVector(zcu)); + const elem_ty = spec.type.childType(zcu); + assert(cg.floatBits(elem_ty) != null); + const ref_ty = splat_spec.ref.typeOf(s); + assert(ref_ty.isVector(zcu) and ref_ty.childType(zcu).toIntern() == elem_ty.toIntern()); + var elem_buf: [@divExact(64, 2)]InternPool.Index = undefined; + const elems = elem_buf[0..spec.type.vectorLen(zcu)]; + const inside_len = (ref_ty.vectorLen(zcu) - 1) % elems.len + 1; + @memset(elems[0..inside_len], (try pt.floatValue(elem_ty, @as(f16, switch (splat_spec.inside) { + .zero => 0.0, + }))).toIntern()); + @memset(elems[inside_len..], (try pt.floatValue(elem_ty, splat_spec.outside)).toIntern()); + return .{ try cg.tempMemFromValue(try pt.aggregateValue(spec.type, elems)), true }; + }, + .frame => |frame_index| .{ try cg.tempInit(spec.type, .{ .load_frame = .{ + .index = frame_index, + .off = switch (frame_index) { + .call_frame => switch (cg.target.cCallingConvention().?) { + .x86_64_sysv => 0, + .x86_64_win => win64_shadow_space, + else => unreachable, + }, + else => 0, + }, + } }), true }, + .lazy_sym => |lazy_symbol_spec| { + const ip = &pt.zcu.intern_pool; + const ty = if (lazy_symbol_spec.ref == .none) spec.type else lazy_symbol_spec.ref.typeOf(s); + return .{ try cg.tempInit(.usize, .{ .lea_lazy_sym = .{ + .kind = lazy_symbol_spec.kind, + .ty = switch (ip.indexToKey(ty.toIntern())) { + .inferred_error_set_type => |func_index| switch (ip.funcIesResolvedUnordered(func_index)) { + .none => unreachable, + else => |ty_index| ty_index, + }, + else => ty.toIntern(), + }, + } }), true }; + }, + .extern_func => |extern_func_spec| .{ try cg.tempInit(spec.type, .{ .lea_extern_func = try cg.addString(std.mem.span(extern_func_spec)) }), true }, + }; + } + + fn finish(spec: TempSpec, temp: *Temp, cg: *CodeGen) InnerError!void { + switch (spec.kind) { + else => {}, + inline .rc_mask, .mut_rc_mask, .ref_mask => |mask| temp.asMask(mask.info, cg), + } + cg.temp_type[@intFromEnum(temp.unwrap(cg).temp)] = spec.type; + } + }; + + const Instruction = struct { + Label, + Mir.Inst.Fixes, + Mir.Inst.Tag, + Select.Operand, + Select.Operand, + Select.Operand, + Select.Operand, + }; + const Label = enum { @"0:", @"1:", @"2:", @"3:", @"4:", @"_", pseudo }; + const Operand = struct { + flags: packed struct(u32) { + tag: Tag, + adjust: Adjust = .none, + base: Ref.Sized = .none, + index: packed struct(u7) { + ref: Ref, + scale: Memory.Scale = .@"1", + } = .{ .ref = .none }, + unused: u2 = 0, + }, + imm: i32 = 0, + + const Tag = enum(u3) { + none, + label, + ref, + simm, + uimm, + lea, + mem, + }; + const Adjust = packed struct(u11) { + sign: enum(u1) { neg, pos }, + lhs: enum(u6) { + none, + ptr_size, + ptr_bit_size, + size, + src0_size, + dst0_size, + delta_size, + delta_elem_size, + unaligned_size, + unaligned_size_add_elem_size, + unaligned_size_sub_elem_size, + unaligned_size_sub_2_elem_size, + bit_size, + src0_bit_size, + @"8_size_sub_bit_size", + len, + elem_limbs, + elem_size, + src0_elem_size, + dst0_elem_size, + src0_elem_size_mul_src1, + vector_index, + src1, + src1_sub_bit_size, + log2_src0_elem_size, + elem_mask, + smin, + smax, + umax, + smin_shr_src1, + smax_shr_src1, + umax_shr_src1, + repeat, + }, + op: enum(u2) { mul, div, div_8_down, rem_8_mul }, + rhs: Memory.Scale, + + const none: Adjust = .{ .sign = .pos, .lhs = .none, .op = .mul, .rhs = .@"1" }; + const sub_ptr_size: Adjust = .{ .sign = .neg, .lhs = .ptr_size, .op = .mul, .rhs = .@"1" }; + const add_ptr_bit_size: Adjust = .{ .sign = .pos, .lhs = .ptr_bit_size, .op = .mul, .rhs = .@"1" }; + const add_size: Adjust = .{ .sign = .pos, .lhs = .size, .op = .mul, .rhs = .@"1" }; + const add_size_div_4: Adjust = .{ .sign = .pos, .lhs = .size, .op = .div, .rhs = .@"4" }; + const add_size_div_8: Adjust = .{ .sign = .pos, .lhs = .size, .op = .div, .rhs = .@"8" }; + const sub_size_div_8: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"8" }; + const sub_size_div_4: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"4" }; + const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" }; + const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" }; + const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; + const add_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; + const add_8_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"8" }; + const add_dst0_size: Adjust = .{ .sign = .pos, .lhs = .dst0_size, .op = .mul, .rhs = .@"1" }; + const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" }; + const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" }; + const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" }; + const add_unaligned_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" }; + const sub_unaligned_size: Adjust = .{ .sign = .neg, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" }; + const add_unaligned_size_add_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_add_elem_size, .op = .mul, .rhs = .@"1" }; + const add_unaligned_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_elem_size, .op = .mul, .rhs = .@"1" }; + const add_unaligned_size_sub_2_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_2_elem_size, .op = .mul, .rhs = .@"1" }; + const add_2_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"2" }; + const add_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"1" }; + const add_bit_size_rem_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"1" }; + const add_bit_size_rem_64: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"8" }; + const add_bit_size_div_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div, .rhs = .@"8" }; + const add_bit_size_div_8_down_1: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div_8_down, .rhs = .@"1" }; + const add_bit_size_div_8_down_2: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div_8_down, .rhs = .@"2" }; + const add_bit_size_div_8_down_8: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .div_8_down, .rhs = .@"8" }; + const sub_bit_size_rem_64: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .rem_8_mul, .rhs = .@"8" }; + const sub_bit_size: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .mul, .rhs = .@"1" }; + const add_src0_bit_size: Adjust = .{ .sign = .pos, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" }; + const sub_src0_bit_size: Adjust = .{ .sign = .neg, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" }; + const add_bit_size_sub_8_size: Adjust = .{ .sign = .neg, .lhs = .@"8_size_sub_bit_size", .op = .mul, .rhs = .@"1" }; + const add_8_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"8" }; + const add_4_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"4" }; + const add_3_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"3" }; + const add_2_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"2" }; + const add_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"1" }; + const sub_len: Adjust = .{ .sign = .neg, .lhs = .len, .op = .mul, .rhs = .@"1" }; + const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" }; + const add_elem_size: Adjust = .{ .sign = .pos, .lhs = .elem_size, .op = .mul, .rhs = .@"1" }; + const add_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .elem_size, .op = .div, .rhs = .@"8" }; + const sub_elem_size_div_8: Adjust = .{ .sign = .neg, .lhs = .elem_size, .op = .div, .rhs = .@"8" }; + const sub_elem_size_div_4: Adjust = .{ .sign = .neg, .lhs = .elem_size, .op = .div, .rhs = .@"4" }; + const add_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; + const add_2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"2" }; + const add_4_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"4" }; + const add_8_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"8" }; + const add_src0_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .div, .rhs = .@"8" }; + const sub_src0_elem_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .div, .rhs = .@"8" }; + const sub_src0_elem_size: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; + const add_src0_elem_size_mul_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; + const sub_src0_elem_size_mul_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; + const add_vector_index: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .mul, .rhs = .@"1" }; + const add_vector_index_rem_32: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .rem_8_mul, .rhs = .@"4" }; + const add_vector_index_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .div_8_down, .rhs = .@"4" }; + const add_dst0_elem_size: Adjust = .{ .sign = .pos, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; + const sub_dst0_elem_size: Adjust = .{ .sign = .neg, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; + const add_src1_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .div_8_down, .rhs = .@"4" }; + const add_src1: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .mul, .rhs = .@"1" }; + const add_src1_rem_32: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"4" }; + const add_src1_rem_64: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"8" }; + const add_src1_sub_bit_size: Adjust = .{ .sign = .pos, .lhs = .src1_sub_bit_size, .op = .mul, .rhs = .@"1" }; + const add_log2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .log2_src0_elem_size, .op = .mul, .rhs = .@"1" }; + const elem_mask: Adjust = .{ .sign = .pos, .lhs = .elem_mask, .op = .mul, .rhs = .@"1" }; + const add_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"1" }; + const add_2_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"2" }; + const sub_smin: Adjust = .{ .sign = .neg, .lhs = .smin, .op = .mul, .rhs = .@"1" }; + const add_smax: Adjust = .{ .sign = .pos, .lhs = .smax, .op = .mul, .rhs = .@"1" }; + const add_umax: Adjust = .{ .sign = .pos, .lhs = .umax, .op = .mul, .rhs = .@"1" }; + const sub_umax: Adjust = .{ .sign = .neg, .lhs = .umax, .op = .mul, .rhs = .@"1" }; + const add_smax_shr_src1: Adjust = .{ .sign = .pos, .lhs = .smax_shr_src1, .op = .mul, .rhs = .@"1" }; + const add_umax_shr_src1: Adjust = .{ .sign = .pos, .lhs = .umax_shr_src1, .op = .mul, .rhs = .@"1" }; + const repeat: Adjust = .{ .sign = .pos, .lhs = .repeat, .op = .mul, .rhs = .@"1" }; + }; + const Ref = enum(u5) { + tmp0, + tmp1, + tmp2, + tmp3, + tmp4, + tmp5, + tmp6, + tmp7, + tmp8, + tmp9, + tmp10, + dst0, + dst1, + src0, + src1, + src2, + none, + + const Sized = packed struct(u9) { + ref: Ref, + size: Memory.Size, + + const none: Sized = .{ .ref = .none, .size = .none }; + + const tmp0: Sized = .{ .ref = .tmp0, .size = .none }; + const tmp0l: Sized = .{ .ref = .tmp0, .size = .low_byte }; + const tmp0h: Sized = .{ .ref = .tmp0, .size = .high_byte }; + const tmp0b: Sized = .{ .ref = .tmp0, .size = .byte }; + const tmp0w: Sized = .{ .ref = .tmp0, .size = .word }; + const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword }; + const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr }; + const tmp0g: Sized = .{ .ref = .tmp0, .size = .gpr }; + const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword }; + const tmp0t: Sized = .{ .ref = .tmp0, .size = .tbyte }; + const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword }; + const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword }; + + const tmp1: Sized = .{ .ref = .tmp1, .size = .none }; + const tmp1l: Sized = .{ .ref = .tmp1, .size = .low_byte }; + const tmp1h: Sized = .{ .ref = .tmp1, .size = .high_byte }; + const tmp1b: Sized = .{ .ref = .tmp1, .size = .byte }; + const tmp1w: Sized = .{ .ref = .tmp1, .size = .word }; + const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword }; + const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr }; + const tmp1g: Sized = .{ .ref = .tmp1, .size = .gpr }; + const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword }; + const tmp1t: Sized = .{ .ref = .tmp1, .size = .tbyte }; + const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword }; + const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword }; + + const tmp2: Sized = .{ .ref = .tmp2, .size = .none }; + const tmp2l: Sized = .{ .ref = .tmp2, .size = .low_byte }; + const tmp2h: Sized = .{ .ref = .tmp2, .size = .high_byte }; + const tmp2b: Sized = .{ .ref = .tmp2, .size = .byte }; + const tmp2w: Sized = .{ .ref = .tmp2, .size = .word }; + const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword }; + const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr }; + const tmp2g: Sized = .{ .ref = .tmp2, .size = .gpr }; + const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword }; + const tmp2t: Sized = .{ .ref = .tmp2, .size = .tbyte }; + const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword }; + const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword }; + + const tmp3: Sized = .{ .ref = .tmp3, .size = .none }; + const tmp3l: Sized = .{ .ref = .tmp3, .size = .low_byte }; + const tmp3h: Sized = .{ .ref = .tmp3, .size = .high_byte }; + const tmp3b: Sized = .{ .ref = .tmp3, .size = .byte }; + const tmp3w: Sized = .{ .ref = .tmp3, .size = .word }; + const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword }; + const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr }; + const tmp3g: Sized = .{ .ref = .tmp3, .size = .gpr }; + const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword }; + const tmp3t: Sized = .{ .ref = .tmp3, .size = .tbyte }; + const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword }; + const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword }; + + const tmp4: Sized = .{ .ref = .tmp4, .size = .none }; + const tmp4l: Sized = .{ .ref = .tmp4, .size = .low_byte }; + const tmp4h: Sized = .{ .ref = .tmp4, .size = .high_byte }; + const tmp4b: Sized = .{ .ref = .tmp4, .size = .byte }; + const tmp4w: Sized = .{ .ref = .tmp4, .size = .word }; + const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword }; + const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr }; + const tmp4g: Sized = .{ .ref = .tmp4, .size = .gpr }; + const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword }; + const tmp4t: Sized = .{ .ref = .tmp4, .size = .tbyte }; + const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword }; + const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword }; + + const tmp5: Sized = .{ .ref = .tmp5, .size = .none }; + const tmp5l: Sized = .{ .ref = .tmp5, .size = .low_byte }; + const tmp5h: Sized = .{ .ref = .tmp5, .size = .high_byte }; + const tmp5b: Sized = .{ .ref = .tmp5, .size = .byte }; + const tmp5w: Sized = .{ .ref = .tmp5, .size = .word }; + const tmp5d: Sized = .{ .ref = .tmp5, .size = .dword }; + const tmp5p: Sized = .{ .ref = .tmp5, .size = .ptr }; + const tmp5g: Sized = .{ .ref = .tmp5, .size = .gpr }; + const tmp5q: Sized = .{ .ref = .tmp5, .size = .qword }; + const tmp5t: Sized = .{ .ref = .tmp5, .size = .tbyte }; + const tmp5x: Sized = .{ .ref = .tmp5, .size = .xword }; + const tmp5y: Sized = .{ .ref = .tmp5, .size = .yword }; + + const tmp6: Sized = .{ .ref = .tmp6, .size = .none }; + const tmp6l: Sized = .{ .ref = .tmp6, .size = .low_byte }; + const tmp6h: Sized = .{ .ref = .tmp6, .size = .high_byte }; + const tmp6b: Sized = .{ .ref = .tmp6, .size = .byte }; + const tmp6w: Sized = .{ .ref = .tmp6, .size = .word }; + const tmp6d: Sized = .{ .ref = .tmp6, .size = .dword }; + const tmp6p: Sized = .{ .ref = .tmp6, .size = .ptr }; + const tmp6g: Sized = .{ .ref = .tmp6, .size = .gpr }; + const tmp6q: Sized = .{ .ref = .tmp6, .size = .qword }; + const tmp6t: Sized = .{ .ref = .tmp6, .size = .tbyte }; + const tmp6x: Sized = .{ .ref = .tmp6, .size = .xword }; + const tmp6y: Sized = .{ .ref = .tmp6, .size = .yword }; + + const tmp7: Sized = .{ .ref = .tmp7, .size = .none }; + const tmp7l: Sized = .{ .ref = .tmp7, .size = .low_byte }; + const tmp7h: Sized = .{ .ref = .tmp7, .size = .high_byte }; + const tmp7b: Sized = .{ .ref = .tmp7, .size = .byte }; + const tmp7w: Sized = .{ .ref = .tmp7, .size = .word }; + const tmp7d: Sized = .{ .ref = .tmp7, .size = .dword }; + const tmp7p: Sized = .{ .ref = .tmp7, .size = .ptr }; + const tmp7g: Sized = .{ .ref = .tmp7, .size = .gpr }; + const tmp7q: Sized = .{ .ref = .tmp7, .size = .qword }; + const tmp7t: Sized = .{ .ref = .tmp7, .size = .tbyte }; + const tmp7x: Sized = .{ .ref = .tmp7, .size = .xword }; + const tmp7y: Sized = .{ .ref = .tmp7, .size = .yword }; + + const tmp8: Sized = .{ .ref = .tmp8, .size = .none }; + const tmp8l: Sized = .{ .ref = .tmp8, .size = .low_byte }; + const tmp8h: Sized = .{ .ref = .tmp8, .size = .high_byte }; + const tmp8b: Sized = .{ .ref = .tmp8, .size = .byte }; + const tmp8w: Sized = .{ .ref = .tmp8, .size = .word }; + const tmp8d: Sized = .{ .ref = .tmp8, .size = .dword }; + const tmp8p: Sized = .{ .ref = .tmp8, .size = .ptr }; + const tmp8g: Sized = .{ .ref = .tmp8, .size = .gpr }; + const tmp8q: Sized = .{ .ref = .tmp8, .size = .qword }; + const tmp8t: Sized = .{ .ref = .tmp8, .size = .tbyte }; + const tmp8x: Sized = .{ .ref = .tmp8, .size = .xword }; + const tmp8y: Sized = .{ .ref = .tmp8, .size = .yword }; + + const tmp9: Sized = .{ .ref = .tmp9, .size = .none }; + const tmp9l: Sized = .{ .ref = .tmp9, .size = .low_byte }; + const tmp9h: Sized = .{ .ref = .tmp9, .size = .high_byte }; + const tmp9b: Sized = .{ .ref = .tmp9, .size = .byte }; + const tmp9w: Sized = .{ .ref = .tmp9, .size = .word }; + const tmp9d: Sized = .{ .ref = .tmp9, .size = .dword }; + const tmp9p: Sized = .{ .ref = .tmp9, .size = .ptr }; + const tmp9g: Sized = .{ .ref = .tmp9, .size = .gpr }; + const tmp9q: Sized = .{ .ref = .tmp9, .size = .qword }; + const tmp9t: Sized = .{ .ref = .tmp9, .size = .tbyte }; + const tmp9x: Sized = .{ .ref = .tmp9, .size = .xword }; + const tmp9y: Sized = .{ .ref = .tmp9, .size = .yword }; + + const tmp10: Sized = .{ .ref = .tmp10, .size = .none }; + const tmp10l: Sized = .{ .ref = .tmp10, .size = .low_byte }; + const tmp10h: Sized = .{ .ref = .tmp10, .size = .high_byte }; + const tmp10b: Sized = .{ .ref = .tmp10, .size = .byte }; + const tmp10w: Sized = .{ .ref = .tmp10, .size = .word }; + const tmp10d: Sized = .{ .ref = .tmp10, .size = .dword }; + const tmp10p: Sized = .{ .ref = .tmp10, .size = .ptr }; + const tmp10g: Sized = .{ .ref = .tmp10, .size = .gpr }; + const tmp10q: Sized = .{ .ref = .tmp10, .size = .qword }; + const tmp10t: Sized = .{ .ref = .tmp10, .size = .tbyte }; + const tmp10x: Sized = .{ .ref = .tmp10, .size = .xword }; + const tmp10y: Sized = .{ .ref = .tmp10, .size = .yword }; + + const dst0: Sized = .{ .ref = .dst0, .size = .none }; + const dst0l: Sized = .{ .ref = .dst0, .size = .low_byte }; + const dst0h: Sized = .{ .ref = .dst0, .size = .high_byte }; + const dst0b: Sized = .{ .ref = .dst0, .size = .byte }; + const dst0w: Sized = .{ .ref = .dst0, .size = .word }; + const dst0d: Sized = .{ .ref = .dst0, .size = .dword }; + const dst0p: Sized = .{ .ref = .dst0, .size = .ptr }; + const dst0g: Sized = .{ .ref = .dst0, .size = .gpr }; + const dst0q: Sized = .{ .ref = .dst0, .size = .qword }; + const dst0t: Sized = .{ .ref = .dst0, .size = .tbyte }; + const dst0x: Sized = .{ .ref = .dst0, .size = .xword }; + const dst0y: Sized = .{ .ref = .dst0, .size = .yword }; + + const dst1: Sized = .{ .ref = .dst1, .size = .none }; + const dst1l: Sized = .{ .ref = .dst1, .size = .low_byte }; + const dst1h: Sized = .{ .ref = .dst1, .size = .high_byte }; + const dst1b: Sized = .{ .ref = .dst1, .size = .byte }; + const dst1w: Sized = .{ .ref = .dst1, .size = .word }; + const dst1d: Sized = .{ .ref = .dst1, .size = .dword }; + const dst1p: Sized = .{ .ref = .dst1, .size = .ptr }; + const dst1g: Sized = .{ .ref = .dst1, .size = .gpr }; + const dst1q: Sized = .{ .ref = .dst1, .size = .qword }; + const dst1t: Sized = .{ .ref = .dst1, .size = .tbyte }; + const dst1x: Sized = .{ .ref = .dst1, .size = .xword }; + const dst1y: Sized = .{ .ref = .dst1, .size = .yword }; + + const src0: Sized = .{ .ref = .src0, .size = .none }; + const src0l: Sized = .{ .ref = .src0, .size = .low_byte }; + const src0h: Sized = .{ .ref = .src0, .size = .high_byte }; + const src0b: Sized = .{ .ref = .src0, .size = .byte }; + const src0w: Sized = .{ .ref = .src0, .size = .word }; + const src0d: Sized = .{ .ref = .src0, .size = .dword }; + const src0p: Sized = .{ .ref = .src0, .size = .ptr }; + const src0g: Sized = .{ .ref = .src0, .size = .gpr }; + const src0q: Sized = .{ .ref = .src0, .size = .qword }; + const src0t: Sized = .{ .ref = .src0, .size = .tbyte }; + const src0x: Sized = .{ .ref = .src0, .size = .xword }; + const src0y: Sized = .{ .ref = .src0, .size = .yword }; + + const src1: Sized = .{ .ref = .src1, .size = .none }; + const src1l: Sized = .{ .ref = .src1, .size = .low_byte }; + const src1h: Sized = .{ .ref = .src1, .size = .high_byte }; + const src1b: Sized = .{ .ref = .src1, .size = .byte }; + const src1w: Sized = .{ .ref = .src1, .size = .word }; + const src1d: Sized = .{ .ref = .src1, .size = .dword }; + const src1p: Sized = .{ .ref = .src1, .size = .ptr }; + const src1g: Sized = .{ .ref = .src1, .size = .gpr }; + const src1q: Sized = .{ .ref = .src1, .size = .qword }; + const src1t: Sized = .{ .ref = .src1, .size = .tbyte }; + const src1x: Sized = .{ .ref = .src1, .size = .xword }; + const src1y: Sized = .{ .ref = .src1, .size = .yword }; + + const src2: Sized = .{ .ref = .src2, .size = .none }; + const src2l: Sized = .{ .ref = .src2, .size = .low_byte }; + const src2h: Sized = .{ .ref = .src2, .size = .high_byte }; + const src2b: Sized = .{ .ref = .src2, .size = .byte }; + const src2w: Sized = .{ .ref = .src2, .size = .word }; + const src2d: Sized = .{ .ref = .src2, .size = .dword }; + const src2p: Sized = .{ .ref = .src2, .size = .ptr }; + const src2g: Sized = .{ .ref = .src2, .size = .gpr }; + const src2q: Sized = .{ .ref = .src2, .size = .qword }; + const src2t: Sized = .{ .ref = .src2, .size = .tbyte }; + const src2x: Sized = .{ .ref = .src2, .size = .xword }; + const src2y: Sized = .{ .ref = .src2, .size = .yword }; + }; + + fn typeOf(ref: Ref, s: *const Select) Type { + return s.types[@intFromEnum(ref)]; + } + + fn tempOf(ref: Ref, s: *const Select) Temp { + return s.temps[@intFromEnum(ref)]; + } + + fn valueOf(ref: Ref, s: *const Select) MCValue { + return switch (ref) { + .none => .none, + else => s.temps[@intFromEnum(ref)].tracking(s.cg).short, + }; + } + }; + + const @"_": Select.Operand = .{ .flags = .{ .tag = .none } }; + + const @"0b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 0) }; + const @"0f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 0) }; + const @"1b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 1) }; + const @"1f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 1) }; + const @"2b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 2) }; + const @"2f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 2) }; + const @"3b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 3) }; + const @"3f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 3) }; + const @"4b": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = ~@as(i32, 4) }; + const @"4f": Select.Operand = .{ .flags = .{ .tag = .label }, .imm = @as(i32, 4) }; + + const tmp0l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0l } }; + const tmp0h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0h } }; + const tmp0b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0b } }; + const tmp0w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0w } }; + const tmp0d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0d } }; + const tmp0p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0p } }; + const tmp0g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0g } }; + const tmp0q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0q } }; + const tmp0t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0t } }; + const tmp0x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0x } }; + const tmp0y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0y } }; + const tmp0w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0w }, .imm = 1 }; + const tmp0d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0d }, .imm = 1 }; + const tmp0p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0p }, .imm = 1 }; + const tmp0q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0q }, .imm = 1 }; + const tmp0w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0w }, .imm = 2 }; + const tmp0d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0d }, .imm = 2 }; + const tmp0p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0p }, .imm = 2 }; + const tmp0q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp0q }, .imm = 2 }; + + const tmp1l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1l } }; + const tmp1h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1h } }; + const tmp1b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1b } }; + const tmp1w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1w } }; + const tmp1d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1d } }; + const tmp1p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1p } }; + const tmp1g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1g } }; + const tmp1q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1q } }; + const tmp1t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1t } }; + const tmp1x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1x } }; + const tmp1y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1y } }; + const tmp1w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1w }, .imm = 1 }; + const tmp1d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1d }, .imm = 1 }; + const tmp1p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1p }, .imm = 1 }; + const tmp1q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1q }, .imm = 1 }; + const tmp1w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1w }, .imm = 2 }; + const tmp1d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1d }, .imm = 2 }; + const tmp1p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1p }, .imm = 2 }; + const tmp1q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp1q }, .imm = 2 }; + + const tmp2l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2l } }; + const tmp2h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2h } }; + const tmp2b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2b } }; + const tmp2w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2w } }; + const tmp2d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2d } }; + const tmp2p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2p } }; + const tmp2g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2g } }; + const tmp2q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2q } }; + const tmp2t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2t } }; + const tmp2x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2x } }; + const tmp2y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2y } }; + const tmp2w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2w }, .imm = 1 }; + const tmp2d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2d }, .imm = 1 }; + const tmp2p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2p }, .imm = 1 }; + const tmp2q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2q }, .imm = 1 }; + const tmp2w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2w }, .imm = 2 }; + const tmp2d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2d }, .imm = 2 }; + const tmp2p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2p }, .imm = 2 }; + const tmp2q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp2q }, .imm = 2 }; + + const tmp3l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3l } }; + const tmp3h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3h } }; + const tmp3b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3b } }; + const tmp3w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3w } }; + const tmp3d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3d } }; + const tmp3p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3p } }; + const tmp3g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3g } }; + const tmp3q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3q } }; + const tmp3t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3t } }; + const tmp3x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3x } }; + const tmp3y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3y } }; + const tmp3w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3w }, .imm = 1 }; + const tmp3d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3d }, .imm = 1 }; + const tmp3p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3p }, .imm = 1 }; + const tmp3q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3q }, .imm = 1 }; + const tmp3w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3w }, .imm = 2 }; + const tmp3d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3d }, .imm = 2 }; + const tmp3p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3p }, .imm = 2 }; + const tmp3q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp3q }, .imm = 2 }; + + const tmp4l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4l } }; + const tmp4h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4h } }; + const tmp4b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4b } }; + const tmp4w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4w } }; + const tmp4d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4d } }; + const tmp4p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4p } }; + const tmp4g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4g } }; + const tmp4q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4q } }; + const tmp4t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4t } }; + const tmp4x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4x } }; + const tmp4y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4y } }; + const tmp4w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4w }, .imm = 1 }; + const tmp4d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4d }, .imm = 1 }; + const tmp4p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4p }, .imm = 1 }; + const tmp4q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4q }, .imm = 1 }; + const tmp4w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4w }, .imm = 2 }; + const tmp4d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4d }, .imm = 2 }; + const tmp4p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4p }, .imm = 2 }; + const tmp4q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp4q }, .imm = 2 }; + + const tmp5l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5l } }; + const tmp5h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5h } }; + const tmp5b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5b } }; + const tmp5w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5w } }; + const tmp5d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5d } }; + const tmp5p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5p } }; + const tmp5g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5g } }; + const tmp5q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5q } }; + const tmp5t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5t } }; + const tmp5x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5x } }; + const tmp5y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5y } }; + const tmp5w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5w }, .imm = 1 }; + const tmp5d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5d }, .imm = 1 }; + const tmp5p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5p }, .imm = 1 }; + const tmp5q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5q }, .imm = 1 }; + const tmp5w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5w }, .imm = 2 }; + const tmp5d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5d }, .imm = 2 }; + const tmp5p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5p }, .imm = 2 }; + const tmp5q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp5q }, .imm = 2 }; + + const tmp6l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6l } }; + const tmp6h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6h } }; + const tmp6b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6b } }; + const tmp6w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6w } }; + const tmp6d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6d } }; + const tmp6p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6p } }; + const tmp6g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6g } }; + const tmp6q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6q } }; + const tmp6t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6t } }; + const tmp6x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6x } }; + const tmp6y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6y } }; + const tmp6w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6w }, .imm = 1 }; + const tmp6d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6d }, .imm = 1 }; + const tmp6p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6p }, .imm = 1 }; + const tmp6q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6q }, .imm = 1 }; + const tmp6w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6w }, .imm = 2 }; + const tmp6d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6d }, .imm = 2 }; + const tmp6p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6p }, .imm = 2 }; + const tmp6q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp6q }, .imm = 2 }; + + const tmp7l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7l } }; + const tmp7h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7h } }; + const tmp7b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7b } }; + const tmp7w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7w } }; + const tmp7d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7d } }; + const tmp7p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7p } }; + const tmp7g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7g } }; + const tmp7q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7q } }; + const tmp7t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7t } }; + const tmp7x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7x } }; + const tmp7y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7y } }; + const tmp7w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7w }, .imm = 1 }; + const tmp7d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7d }, .imm = 1 }; + const tmp7p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7p }, .imm = 1 }; + const tmp7q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7q }, .imm = 1 }; + const tmp7w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7w }, .imm = 2 }; + const tmp7d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7d }, .imm = 2 }; + const tmp7p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7p }, .imm = 2 }; + const tmp7q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp7q }, .imm = 2 }; + + const tmp8l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8l } }; + const tmp8h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8h } }; + const tmp8b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8b } }; + const tmp8w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8w } }; + const tmp8d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8d } }; + const tmp8p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8p } }; + const tmp8g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8g } }; + const tmp8q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8q } }; + const tmp8t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8t } }; + const tmp8x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8x } }; + const tmp8y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8y } }; + const tmp8w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8w }, .imm = 1 }; + const tmp8d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8d }, .imm = 1 }; + const tmp8p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8p }, .imm = 1 }; + const tmp8q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8q }, .imm = 1 }; + const tmp8w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8w }, .imm = 2 }; + const tmp8d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8d }, .imm = 2 }; + const tmp8p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8p }, .imm = 2 }; + const tmp8q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp8q }, .imm = 2 }; + + const tmp9l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9l } }; + const tmp9h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9h } }; + const tmp9b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9b } }; + const tmp9w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9w } }; + const tmp9d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9d } }; + const tmp9p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9p } }; + const tmp9g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9g } }; + const tmp9q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9q } }; + const tmp9t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9t } }; + const tmp9x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9x } }; + const tmp9y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9y } }; + const tmp9w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9w }, .imm = 1 }; + const tmp9d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9d }, .imm = 1 }; + const tmp9p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9p }, .imm = 1 }; + const tmp9q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9q }, .imm = 1 }; + const tmp9w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9w }, .imm = 2 }; + const tmp9d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9d }, .imm = 2 }; + const tmp9p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9p }, .imm = 2 }; + const tmp9q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp9q }, .imm = 2 }; + + const tmp10l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10l } }; + const tmp10h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10h } }; + const tmp10b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10b } }; + const tmp10w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10w } }; + const tmp10d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10d } }; + const tmp10p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10p } }; + const tmp10g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10g } }; + const tmp10q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10q } }; + const tmp10t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10t } }; + const tmp10x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10x } }; + const tmp10y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10y } }; + const tmp10w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10w }, .imm = 1 }; + const tmp10d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10d }, .imm = 1 }; + const tmp10p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10p }, .imm = 1 }; + const tmp10q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10q }, .imm = 1 }; + const tmp10w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10w }, .imm = 2 }; + const tmp10d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10d }, .imm = 2 }; + const tmp10p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10p }, .imm = 2 }; + const tmp10q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .tmp10q }, .imm = 2 }; + + const dst0l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0l } }; + const dst0h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0h } }; + const dst0b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0b } }; + const dst0w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0w } }; + const dst0d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0d } }; + const dst0p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0p } }; + const dst0g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0g } }; + const dst0q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0q } }; + const dst0t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0t } }; + const dst0x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0x } }; + const dst0y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0y } }; + const dst0w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0w }, .imm = 1 }; + const dst0d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0d }, .imm = 1 }; + const dst0p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0p }, .imm = 1 }; + const dst0q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0q }, .imm = 1 }; + const dst0w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0w }, .imm = 2 }; + const dst0d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0d }, .imm = 2 }; + const dst0p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0p }, .imm = 2 }; + const dst0q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst0q }, .imm = 2 }; + + const dst1l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1l } }; + const dst1h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1h } }; + const dst1b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1b } }; + const dst1w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1w } }; + const dst1d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1d } }; + const dst1p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1p } }; + const dst1g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1g } }; + const dst1q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1q } }; + const dst1t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1t } }; + const dst1x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1x } }; + const dst1y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1y } }; + const dst1w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1w }, .imm = 1 }; + const dst1d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1d }, .imm = 1 }; + const dst1p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1p }, .imm = 1 }; + const dst1q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1q }, .imm = 1 }; + const dst1w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1w }, .imm = 2 }; + const dst1d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1d }, .imm = 2 }; + const dst1p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1p }, .imm = 2 }; + const dst1q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .dst1q }, .imm = 2 }; + + const src0l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0l } }; + const src0h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0h } }; + const src0b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0b } }; + const src0w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0w } }; + const src0d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0d } }; + const src0p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0p } }; + const src0g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0g } }; + const src0q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0q } }; + const src0t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0t } }; + const src0x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0x } }; + const src0y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0y } }; + const src0w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0w }, .imm = 1 }; + const src0d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0d }, .imm = 1 }; + const src0p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0p }, .imm = 1 }; + const src0q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0q }, .imm = 1 }; + const src0w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0w }, .imm = 2 }; + const src0d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0d }, .imm = 2 }; + const src0p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0p }, .imm = 2 }; + const src0q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src0q }, .imm = 2 }; + + const src1l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1l } }; + const src1h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1h } }; + const src1b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1b } }; + const src1w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1w } }; + const src1d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1d } }; + const src1p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1p } }; + const src1g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1g } }; + const src1q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1q } }; + const src1t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1t } }; + const src1x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1x } }; + const src1y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1y } }; + const src1w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1w }, .imm = 1 }; + const src1d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1d }, .imm = 1 }; + const src1p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1p }, .imm = 1 }; + const src1q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1q }, .imm = 1 }; + const src1w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1w }, .imm = 2 }; + const src1d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1d }, .imm = 2 }; + const src1p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1p }, .imm = 2 }; + const src1q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src1q }, .imm = 2 }; + + const src2l: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2l } }; + const src2h: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2h } }; + const src2b: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2b } }; + const src2w: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2w } }; + const src2d: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2d } }; + const src2p: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2p } }; + const src2g: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2g } }; + const src2q: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2q } }; + const src2t: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2t } }; + const src2x: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2x } }; + const src2y: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2y } }; + const src2w0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2w }, .imm = 1 }; + const src2d0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2d }, .imm = 1 }; + const src2p0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2p }, .imm = 1 }; + const src2q0: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2q }, .imm = 1 }; + const src2w1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2w }, .imm = 2 }; + const src2d1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2d }, .imm = 2 }; + const src2p1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2p }, .imm = 2 }; + const src2q1: Select.Operand = .{ .flags = .{ .tag = .ref, .base = .src2q }, .imm = 2 }; + + fn si(imm: i32) Select.Operand { + return .{ .flags = .{ .tag = .simm }, .imm = imm }; + } + fn sa(base: Ref.Sized, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base } }; + } + fn sa2(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base, .index = .{ .ref = index } } }; + } + fn sia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base }, .imm = imm }; + } + fn sia2(imm: i32, base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .simm, .adjust = adjust, .base = base, .index = .{ .ref = index } }, .imm = imm }; + } + fn ui(imm: u32) Select.Operand { + return .{ .flags = .{ .tag = .uimm }, .imm = @bitCast(imm) }; + } + fn ua(base: Ref.Sized, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .uimm, .adjust = adjust, .base = base } }; + } + fn uia(imm: u32, base: Ref.Sized, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .uimm, .adjust = adjust, .base = base }, .imm = @bitCast(imm) }; + } + fn uas(base: Ref.Sized, scale: Memory.Scale, adjust: Adjust) Select.Operand { + return .{ .flags = .{ .tag = .uimm, .adjust = adjust, .base = base, .index = .{ .scale = scale } } }; + } + + fn rm(mode: bits.RoundMode) Select.Operand { + return .{ .flags = .{ .tag = .uimm }, .imm = @intCast(mode.imm().unsigned) }; + } + fn sp(pred: bits.SseFloatPredicate) Select.Operand { + return .{ .flags = .{ .tag = .uimm }, .imm = @intCast(pred.imm().unsigned) }; + } + fn vp(pred: bits.VexFloatPredicate) Select.Operand { + return .{ .flags = .{ .tag = .uimm }, .imm = @intCast(pred.imm().unsigned) }; + } + + fn lea(base: Ref.Sized) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .base = base, + }, + }; + } + fn leaa(base: Ref.Sized, adjust: Adjust) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .adjust = adjust, + .base = base, + }, + }; + } + fn leaad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .adjust = adjust, + .base = base, + }, + .imm = disp, + }; + } + fn lead(base: Ref.Sized, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .base = base, + }, + .imm = disp, + }; + } + fn leai(base: Ref.Sized, index: Ref) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .base = base, + .index = .{ .ref = index }, + }, + }; + } + fn leaia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .adjust = adjust, + .base = base, + .index = .{ .ref = index }, + }, + }; + } + fn leaiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .adjust = adjust, + .base = base, + .index = .{ .ref = index }, + }, + .imm = disp, + }; + } + fn leaid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .base = base, + .index = .{ .ref = index }, + }, + .imm = disp, + }; + } + fn leasi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + }; + } + fn leasia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .adjust = adjust, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + }; + } + fn leasid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + .imm = disp, + }; + } + fn leasiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .lea, + .adjust = adjust, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + .imm = disp, + }; + } + + fn mem(base: Ref.Sized) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .base = base, + }, + }; + } + fn memd(base: Ref.Sized, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .base = base, + }, + .imm = disp, + }; + } + fn mema(base: Ref.Sized, adjust: Adjust) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .adjust = adjust, + .base = base, + }, + }; + } + fn memad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .adjust = adjust, + .base = base, + }, + .imm = disp, + }; + } + fn memi(base: Ref.Sized, index: Ref) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .base = base, + .index = .{ .ref = index }, + }, + }; + } + fn memia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .adjust = adjust, + .base = base, + .index = .{ .ref = index }, + }, + }; + } + fn memiad(base: Ref.Sized, index: Ref, adjust: Adjust, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .adjust = adjust, + .base = base, + .index = .{ .ref = index }, + }, + .imm = disp, + }; + } + fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .base = base, + .index = .{ .ref = index }, + }, + .imm = disp, + }; + } + fn memsi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + }; + } + fn memsia(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .adjust = adjust, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + }; + } + fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + .imm = disp, + }; + } + fn memsiad(base: Ref.Sized, scale: Memory.Scale, index: Ref, adjust: Adjust, disp: i32) Select.Operand { + return .{ + .flags = .{ + .tag = .mem, + .adjust = adjust, + .base = base, + .index = .{ .ref = index, .scale = scale }, + }, + .imm = disp, + }; + } + + fn adjustedImm(op: Select.Operand, comptime SignedImm: type, s: *const Select) SignedImm { + const UnsignedImm = @Type(.{ + .int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits }, + }); + const lhs: SignedImm = lhs: switch (op.flags.adjust.lhs) { + .none => 0, + .ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8), + .ptr_bit_size => s.cg.target.ptrBitWidth(), + .size => @intCast(op.flags.base.ref.typeOf(s).abiSize(s.cg.pt.zcu)), + .src0_size => @intCast(Select.Operand.Ref.src0.typeOf(s).abiSize(s.cg.pt.zcu)), + .dst0_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).abiSize(s.cg.pt.zcu)), + .delta_size => @intCast(@as(SignedImm, @intCast(op.flags.base.ref.typeOf(s).abiSize(s.cg.pt.zcu))) - + @as(SignedImm, @intCast(op.flags.index.ref.typeOf(s).abiSize(s.cg.pt.zcu)))), + .delta_elem_size => @intCast(@as(SignedImm, @intCast(op.flags.base.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))) - + @as(SignedImm, @intCast(op.flags.index.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)))), + .unaligned_size => @intCast(s.cg.unalignedSize(op.flags.base.ref.typeOf(s))), + .unaligned_size_add_elem_size => { + const ty = op.flags.base.ref.typeOf(s); + break :lhs @intCast(s.cg.unalignedSize(ty) + ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)); + }, + .unaligned_size_sub_elem_size => { + const ty = op.flags.base.ref.typeOf(s); + break :lhs @intCast(s.cg.unalignedSize(ty) - ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)); + }, + .unaligned_size_sub_2_elem_size => { + const ty = op.flags.base.ref.typeOf(s); + break :lhs @intCast(s.cg.unalignedSize(ty) - ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * 2); + }, + .bit_size => @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s))), + .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))), + .@"8_size_sub_bit_size" => { + const ty = op.flags.base.ref.typeOf(s); + break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu)); + }, + .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)), + .elem_limbs => @intCast(@divExact( + op.flags.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu), + @divExact(op.flags.base.size.bitSize(s.cg.target), 8), + )), + .elem_size => @intCast(op.flags.base.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), + .src0_elem_size => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), + .dst0_elem_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), + .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * + Select.Operand.Ref.src1.valueOf(s).immediate), + .vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) { + .none, .runtime => unreachable, + else => |vector_index| @intFromEnum(vector_index), + }, + .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), + .src1_sub_bit_size => @as(SignedImm, @intCast(Select.Operand.Ref.src1.valueOf(s).immediate)) - + @as(SignedImm, @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)))), + .log2_src0_elem_size => @intCast(std.math.log2(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))), + .elem_mask => @as(u8, std.math.maxInt(u8)) >> @intCast( + 8 - ((s.cg.unalignedSize(op.flags.base.ref.typeOf(s)) - 1) % + @divExact(op.flags.base.size.bitSize(s.cg.target), 8) + 1 >> + op.flags.index.scale.toLog2()), + ), + inline .smin, .smax, .umax, .smin_shr_src1, .smax_shr_src1, .umax_shr_src1 => |adjust| switch (op.flags.base.size) { + else => unreachable, + inline .none, .byte, .word, .dword, .qword => |size| { + const Imm = switch (adjust) { + else => comptime unreachable, + .smin, .smax, .smin_shr_src1, .smax_shr_src1 => SignedImm, + .umax, .umax_shr_src1 => UnsignedImm, + }; + const RefImm = switch (size) { + else => comptime unreachable, + .none => Imm, + .byte, .word, .dword, .qword => @Type(comptime .{ .int = .{ + .signedness = @typeInfo(Imm).int.signedness, + .bits = size.bitSize(undefined), + } }), + }; + break :lhs @bitCast(@as(Imm, @intCast(@as(RefImm, switch (adjust) { + else => comptime unreachable, + .smin, .smin_shr_src1 => std.math.minInt, + .smax, .umax, .smax_shr_src1, .umax_shr_src1 => std.math.maxInt, + }(RefImm)) >> @truncate(switch (adjust) { + else => comptime unreachable, + .smin, .smax, .umax => 0, + .smin_shr_src1, .smax_shr_src1, .umax_shr_src1 => Select.Operand.Ref.src1.valueOf(s).immediate, + } -% s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)))))); + }, + }, + .repeat => switch (SignedImm) { + else => unreachable, + i64 => return @as(i64, op.imm) << 32 | @as(u32, @bitCast(op.imm)), + }, + }; + const rhs = op.flags.adjust.rhs.toLog2(); + const op_res = op_res: switch (op.flags.adjust.op) { + .mul => { + const op_res = @shlWithOverflow(lhs, rhs); + assert(op_res[1] == 0); + break :op_res op_res[0]; + }, + .div => @shrExact(lhs, rhs), + .div_8_down => lhs >> 3 & @as(SignedImm, -1) << rhs, + .rem_8_mul => lhs & (@as(SignedImm, 1) << @intCast(@as(u3, 3) + rhs)) - 1, + }; + const disp: SignedImm = @bitCast(@as(UnsignedImm, @as(u32, @bitCast(op.imm)))); + return switch (op.flags.adjust.sign) { + .neg => disp - op_res, + .pos => disp + op_res, + }; + } + + fn lower(op: Select.Operand, s: *Select) InnerError!CodeGen.Operand { + return switch (op.flags.tag) { + .none => .none, + .label => if (op.imm < 0) + .{ .inst = s.labels[@intCast(~op.imm)].backward.? } + else for (&s.labels[@intCast(op.imm)].forward) |*label| { + if (label.*) |_| continue; + label.* = @intCast(s.cg.mir_instructions.len); + break .{ .inst = undefined }; + } else unreachable, + .ref => switch (op.imm) { + else => unreachable, + 0 => switch (op.flags.base.ref.valueOf(s)) { + .none, .unreach, .dead, .undef => unreachable, + .immediate => |imm| .{ .imm = switch (op.flags.base.size) { + .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))), + .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))), + .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))), + .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm), + else => unreachable, + } }, + .eflags => unreachable, + .register => |reg| .{ .reg = s.lowerReg(reg.toSize(op.flags.base.size, s.cg.target)) }, + .register_pair, .register_triple, .register_quadruple, .register_offset, .register_overflow => unreachable, + .register_mask => |reg_mask| .{ .reg = s.lowerReg(reg_mask.reg.toSize(op.flags.base.size, s.cg.target)) }, + .lea_nav => |nav| .{ .imm = .{ .nav = .{ .index = nav } } }, + .lea_uav => |uav| .{ .imm = .{ .uav = uav } }, + .lea_lazy_sym => |lazy_sym| .{ .imm = .{ .lazy_sym = lazy_sym } }, + .lea_extern_func => |extern_func| .{ .imm = .{ .extern_func = extern_func } }, + else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.flags.base.size }) }, + .lea_frame, .elementwise_args, .reserved_frame, .air_ref => unreachable, + }, + 1...2 => |imm| switch (op.flags.base.ref.valueOf(s)) { + inline .register_pair, .register_triple, .register_quadruple => |regs| .{ + .reg = s.lowerReg(regs[@intCast(imm - 1)].toSize(op.flags.base.size, s.cg.target)), + }, + else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ + .size = op.flags.base.size, + .disp = @as(i32, @intCast(@divExact(op.flags.base.size.bitSize(s.cg.target), 8))) * (imm - 1), + }) }, + }, + }, + .simm => .{ .imm = .s(op.adjustedImm(i32, s)) }, + .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) }, + .lea => .{ .mem = try op.flags.base.ref.valueOf(s).deref().mem(s.cg, .{ + .size = op.flags.base.size, + .index = switch (op.flags.index.ref.valueOf(s)) { + else => unreachable, + .none => .none, + .register => |index_reg| index_reg.toSize(.ptr, s.cg.target), + }, + .scale = op.flags.index.scale, + .disp = op.adjustedImm(i32, s) + switch (op.flags.index.ref.valueOf(s)) { + else => unreachable, + .none, .register, .lea_nav, .lea_uav, .lea_lazy_sym, .lea_extern_func => 0, + .register_offset => |base_reg_off| base_reg_off.off, + }, + }) }, + .mem => .{ .mem = try op.flags.base.ref.valueOf(s).mem(s.cg, .{ + .size = op.flags.base.size, + .index = switch (op.flags.index.ref.valueOf(s)) { + else => unreachable, + .none => .none, + .register => |index_reg| index_reg.toSize(.ptr, s.cg.target), + }, + .scale = op.flags.index.scale, + .disp = op.adjustedImm(i32, s), + }) }, + }; + } + }; +}; +fn select( + cg: *CodeGen, + dst_temps: []Temp, + dst_tys: []const Type, + src_temps: []Temp, + cases: []const Select.Case, +) Select.Error!void { + @setEvalBranchQuota(33_500); + cases: for (cases) |case| { + switch (case.required_abi) { + .any => {}, + .gnu => if (switch (cg.target.abi) { + .none, .msvc, .itanium => cg.target.os.tag == .windows, + else => false, + }) continue :cases, + .msvc => if (switch (cg.target.abi) { + .none, .msvc, .itanium => cg.target.os.tag != .windows, + else => true, + }) continue :cases, + } + switch (case.required_cc_abi) { + .any => {}, + .sysv64 => if (cg.target.cCallingConvention().? != .x86_64_sysv) continue :cases, + .win64 => if (cg.target.cCallingConvention().? != .x86_64_win) continue :cases, + } + for (case.required_features) |required_feature| if (required_feature) |feature| if (!cg.hasFeature(feature)) continue :cases; + for (case.src_constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases; + for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases; + if (std.debug.runtime_safety) { + for (case.src_constraints[src_temps.len..]) |src_constraint| assert(src_constraint == .any); + for (case.dst_constraints[dst_temps.len..]) |dst_constraint| assert(dst_constraint == .any); + for (case.dst_temps[dst_temps.len..]) |dst_temp| assert(dst_temp == .unused); + } + patterns: for (case.patterns) |pattern| { + for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; + if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) |src_pattern| assert(src_pattern == .none); + + if (case.call_frame.alignment != .none) { + const frame_allocs_slice = cg.frame_allocs.slice(); + const stack_frame_size = + &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_size.* = @max(stack_frame_size.*, switch (cg.target.cCallingConvention().?) { + .x86_64_sysv => case.call_frame.size, + .x86_64_win => win64_shadow_space + case.call_frame.size, + else => unreachable, + }); + const stack_frame_align = + &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_align.* = stack_frame_align.max(case.call_frame.alignment); + } + + var s: Select = .{ + .cg = cg, + .temps = undefined, + .types = undefined, + .labels = @splat(.{ .forward = @splat(null), .backward = null }), + .top = 0, + }; + const s_tmp_types = s.types[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)]; + const s_tmp_temps = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)]; + const s_dst_types = s.types[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; + const s_dst_temps = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; + const s_src_types = s.types[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; + const s_src_temps = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; + + for (s_tmp_types, case.extra_temps) |*ty, spec| ty.* = spec.type; + @memcpy(s_dst_types[0..dst_tys.len], dst_tys); + for (s_src_types[0..src_temps.len], src_temps) |*ty, temp| ty.* = temp.typeOf(cg); + std.mem.swap(Type, &s_src_types[pattern.commute[0]], &s_src_types[pattern.commute[1]]); + + caller_preserved: { + const cc = switch (case.clobbers.caller_preserved) { + .none => break :caller_preserved, + .ccc => cg.target.cCallingConvention().?, + .zigcc => .auto, + }; + assert(case.clobbers.eflags); + const err_ret_trace_reg = if (cc == .auto and cg.pt.zcu.comp.config.any_error_tracing) err_ret_trace_reg: { + const param_gpr = abi.getCAbiIntParamRegs(.auto); + break :err_ret_trace_reg param_gpr[param_gpr.len - 1]; + } else .none; + switch (cc) { + else => unreachable, + inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| skip: { + if (reg == err_ret_trace_reg) break :skip; + const tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse break :skip; + try cg.register_manager.getRegIndex(tracked_index, null); + assert(cg.register_manager.lockRegIndexAssumeUnused(tracked_index).tracked_index == tracked_index); + }, + } + } + + @memcpy(s_src_temps[0..src_temps.len], src_temps); + std.mem.swap(Temp, &s_src_temps[pattern.commute[0]], &s_src_temps[pattern.commute[1]]); + var dst_locks: [s_dst_temps.len][2]?RegisterLock = @splat(@splat(null)); + for (dst_locks[0..dst_temps.len], case.dst_temps[0..dst_temps.len]) |*dst_lock, dst_kind| dst_lock.* = try dst_kind.lock(cg); + var tmp_locks: [s_tmp_temps.len][2]?RegisterLock = @splat(@splat(null)); + for (&tmp_locks, case.extra_temps) |*tmp_lock, tmp_spec| tmp_lock.* = try tmp_spec.kind.lock(cg); + + while (true) for (pattern.src[0..src_temps.len], src_temps) |src_pattern, *src_temp| { + if (try src_pattern.convert(src_temp, cg)) break; + } else break; + var src_locks: [s_src_temps.len][2]?RegisterLock = @splat(@splat(null)); + for (src_locks[0..src_temps.len], src_temps) |*locks, src_temp| { + const regs: [2]Register = switch (src_temp.tracking(cg).short) { + else => continue, + .register => |reg| .{ reg, .none }, + .register_pair => |regs| regs, + }; + for (regs, locks) |reg, *lock| { + if (reg == .none) continue; + lock.* = cg.register_manager.lockRegIndex(RegisterManager.indexOfRegIntoTracked(reg) orelse continue); + } + } + @memcpy(s_src_temps[0..src_temps.len], src_temps); + std.mem.swap(Temp, &s_src_temps[pattern.commute[0]], &s_src_temps[pattern.commute[1]]); + + if (case.clobbers.eflags) try cg.spillEflagsIfOccupied(); + + var tmp_owned: [s_tmp_temps.len]bool = @splat(false); + for (s_tmp_temps, &tmp_owned, case.extra_temps) |*temp, *owned, tmp_spec| temp.*, owned.* = try tmp_spec.create(&s); + for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, tmp_kind| dst_temp.*, _ = try Select.TempSpec.create(.{ .type = dst_ty, .kind = tmp_kind }, &s); + @memcpy(s_dst_temps[0..dst_temps.len], dst_temps); + + switch (case.each) { + .once => |body| { + for (body) |inst| try s.emit(inst); + s.emitLabel(.@"0:"); + }, + } + assert(s.top == 0); + + for (src_locks) |locks| for (locks) |lock| if (lock) |reg| cg.register_manager.unlockReg(reg); + for (tmp_locks) |locks| for (locks) |lock| if (lock) |reg| cg.register_manager.unlockReg(reg); + for (dst_locks) |locks| for (locks) |lock| if (lock) |reg| cg.register_manager.unlockReg(reg); + caller_preserved: { + const cc = switch (case.clobbers.caller_preserved) { + .none => break :caller_preserved, + .ccc => cg.target.cCallingConvention().?, + .zigcc => .auto, + }; + const err_ret_trace_reg = if (cc == .auto and cg.pt.zcu.comp.config.any_error_tracing) err_ret_trace_reg: { + const param_gpr = abi.getCAbiIntParamRegs(.auto); + break :err_ret_trace_reg param_gpr[param_gpr.len - 1]; + } else .none; + switch (cc) { + else => unreachable, + inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| skip: { + if (reg == err_ret_trace_reg) break :skip; + cg.register_manager.unlockReg(.{ .tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse break :skip }); + }, + } + } + for (tmp_owned, s_tmp_temps) |owned, temp| if (owned) try temp.die(cg); + for (dst_tys, case.dst_temps[0..dst_temps.len], dst_temps) |dst_ty, dst_kind, *dst_temp| try Select.TempSpec.finish(.{ .type = dst_ty, .kind = dst_kind }, dst_temp, cg); + return; + } + } + return error.SelectFailed; +} diff --git a/src/codegen/x86_64/Disassembler.zig b/src/codegen/x86_64/Disassembler.zig new file mode 100644 index 0000000000000000000000000000000000000000..7aeb06f9638fccd2db163c4ad137ee228c455cfb --- /dev/null +++ b/src/codegen/x86_64/Disassembler.zig @@ -0,0 +1,509 @@ +const Disassembler = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const math = std.math; + +const bits = @import("bits.zig"); +const encoder = @import("encoder.zig"); + +const Encoding = @import("Encoding.zig"); +const Immediate = Instruction.Immediate; +const Instruction = encoder.Instruction; +const LegacyPrefixes = encoder.LegacyPrefixes; +const Memory = Instruction.Memory; +const Register = bits.Register; +const Rex = encoder.Rex; + +pub const Error = error{ + EndOfStream, + /// After the TODO below is solved this will make sense. + ReadFailed, + LegacyPrefixAfterRex, + UnknownOpcode, + Overflow, + Todo, +}; + +// TODO these fields should be replaced by std.Io.Reader + +code: []const u8, +pos: usize = 0, + +pub fn init(code: []const u8) Disassembler { + return .{ .code = code }; +} + +pub fn next(dis: *Disassembler) Error!?Instruction { + const prefixes = dis.parsePrefixes() catch |err| switch (err) { + error.EndOfStream => return null, + else => |e| return e, + }; + + const enc = try dis.parseEncoding(prefixes) orelse return error.UnknownOpcode; + switch (enc.data.op_en) { + .z => return inst(enc, .{}), + .o => { + const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, + }); + }, + .zo => { + const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); + return inst(enc, .{ + .op1 = .{ .reg = enc.data.ops[0].toReg() }, + .op2 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) }, + }); + }, + .oz => { + const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, + .op2 = .{ .reg = enc.data.ops[1].toReg() }, + }); + }, + .oi => { + const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]); + const imm = try dis.parseImm(enc.data.ops[1]); + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, + .op2 = .{ .imm = imm }, + }); + }, + .i, .d => { + const imm = try dis.parseImm(enc.data.ops[0]); + return inst(enc, .{ + .op1 = .{ .imm = imm }, + }); + }, + .zi => { + const imm = try dis.parseImm(enc.data.ops[1]); + return inst(enc, .{ + .op1 = .{ .reg = enc.data.ops[0].toReg() }, + .op2 = .{ .imm = imm }, + }); + }, + .ii => { + const imm1 = try dis.parseImm(enc.data.ops[0]); + const imm2 = try dis.parseImm(enc.data.ops[1]); + return inst(enc, .{ + .op1 = .{ .imm = imm1 }, + .op2 = .{ .imm = imm2 }, + }); + }, + .ia => { + const imm = try dis.parseImm(enc.data.ops[0]); + return inst(enc, .{ + .op1 = .{ .imm = imm }, + .op2 = .{ .reg = .eax }, + }); + }, + .m, .mi, .m1, .mc => { + const modrm = try dis.parseModRmByte(); + const act_enc = Encoding.findByOpcode(enc.opcode(), .{ + .legacy = prefixes.legacy, + .rex = prefixes.rex, + }, modrm.op1) orelse return error.UnknownOpcode; + const sib = if (modrm.sib()) try dis.parseSibByte() else null; + + if (modrm.direct()) { + const op2: Instruction.Operand = switch (act_enc.data.op_en) { + .mi => .{ .imm = try dis.parseImm(act_enc.data.ops[1]) }, + .m1 => .{ .imm = Immediate.u(1) }, + .mc => .{ .reg = .cl }, + .m => .none, + else => unreachable, + }; + return inst(act_enc, .{ + .op1 = .{ .reg = parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, act_enc.data.ops[0].regBitSize()) }, + .op2 = op2, + }); + } + + const disp = try dis.parseDisplacement(modrm, sib); + const op2: Instruction.Operand = switch (act_enc.data.op_en) { + .mi => .{ .imm = try dis.parseImm(act_enc.data.ops[1]) }, + .m1 => .{ .imm = Immediate.u(1) }, + .mc => .{ .reg = .cl }, + .m => .none, + else => unreachable, + }; + + if (modrm.rip()) { + return inst(act_enc, .{ + .op1 = .{ .mem = Memory.initRip(Memory.PtrSize.fromBitSize(act_enc.data.ops[0].memBitSize()), disp) }, + .op2 = op2, + }); + } + + const scale_index = if (sib) |info| info.scaleIndex(prefixes.rex) else null; + const base = if (sib) |info| + info.baseReg(modrm, prefixes) + else + parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, 64); + return inst(act_enc, .{ + .op1 = .{ .mem = Memory.initSib(Memory.PtrSize.fromBitSize(act_enc.data.ops[0].memBitSize()), .{ + .base = if (base) |base_reg| .{ .reg = base_reg } else .none, + .scale_index = scale_index, + .disp = disp, + }) }, + .op2 = op2, + }); + }, + .fd => { + const seg = segmentRegister(prefixes.legacy); + const offset = try dis.parseOffset(); + return inst(enc, .{ + .op1 = .{ .reg = enc.data.ops[0].toReg() }, + .op2 = .{ .mem = Memory.initMoffs(seg, offset) }, + }); + }, + .td => { + const seg = segmentRegister(prefixes.legacy); + const offset = try dis.parseOffset(); + return inst(enc, .{ + .op1 = .{ .mem = Memory.initMoffs(seg, offset) }, + .op2 = .{ .reg = enc.data.ops[1].toReg() }, + }); + }, + .mr, .mri, .mrc => { + const modrm = try dis.parseModRmByte(); + const sib = if (modrm.sib()) try dis.parseSibByte() else null; + const src_bit_size = enc.data.ops[1].regBitSize(); + + if (modrm.direct()) { + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) }, + .op2 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.x, prefixes.rex, src_bit_size) }, + }); + } + + const dst_bit_size = enc.data.ops[0].memBitSize(); + const disp = try dis.parseDisplacement(modrm, sib); + const op3: Instruction.Operand = switch (enc.data.op_en) { + .mri => .{ .imm = try dis.parseImm(enc.data.ops[2]) }, + .mrc => .{ .reg = .cl }, + .mr => .none, + else => unreachable, + }; + + if (modrm.rip()) { + return inst(enc, .{ + .op1 = .{ .mem = Memory.initRip(Memory.PtrSize.fromBitSize(dst_bit_size), disp) }, + .op2 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, src_bit_size) }, + .op3 = op3, + }); + } + + const scale_index = if (sib) |info| info.scaleIndex(prefixes.rex) else null; + const base = if (sib) |info| + info.baseReg(modrm, prefixes) + else + parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, 64); + return inst(enc, .{ + .op1 = .{ .mem = Memory.initSib(Memory.PtrSize.fromBitSize(dst_bit_size), .{ + .base = if (base) |base_reg| .{ .reg = base_reg } else .none, + .scale_index = scale_index, + .disp = disp, + }) }, + .op2 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, src_bit_size) }, + .op3 = op3, + }); + }, + .rm, .rmi => { + const modrm = try dis.parseModRmByte(); + const sib = if (modrm.sib()) try dis.parseSibByte() else null; + const dst_bit_size = enc.data.ops[0].regBitSize(); + + if (modrm.direct()) { + const op3: Instruction.Operand = switch (enc.data.op_en) { + .rm => .none, + .rmi => .{ .imm = try dis.parseImm(enc.data.ops[2]) }, + else => unreachable, + }; + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.x, prefixes.rex, dst_bit_size) }, + .op2 = .{ .reg = parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) }, + .op3 = op3, + }); + } + + const src_bit_size = if (enc.data.ops[1] == .m) dst_bit_size else enc.data.ops[1].memBitSize(); + const disp = try dis.parseDisplacement(modrm, sib); + const op3: Instruction.Operand = switch (enc.data.op_en) { + .rmi => .{ .imm = try dis.parseImm(enc.data.ops[2]) }, + .rm => .none, + else => unreachable, + }; + + if (modrm.rip()) { + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, dst_bit_size) }, + .op2 = .{ .mem = Memory.initRip(Memory.PtrSize.fromBitSize(src_bit_size), disp) }, + .op3 = op3, + }); + } + + const scale_index = if (sib) |info| info.scaleIndex(prefixes.rex) else null; + const base = if (sib) |info| + info.baseReg(modrm, prefixes) + else + parseGpRegister(modrm.op2, prefixes.rex.b, prefixes.rex, 64); + return inst(enc, .{ + .op1 = .{ .reg = parseGpRegister(modrm.op1, prefixes.rex.r, prefixes.rex, dst_bit_size) }, + .op2 = .{ .mem = Memory.initSib(Memory.PtrSize.fromBitSize(src_bit_size), .{ + .base = if (base) |base_reg| .{ .reg = base_reg } else .none, + .scale_index = scale_index, + .disp = disp, + }) }, + .op3 = op3, + }); + }, + .rm0, .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable, // TODO + } +} + +fn inst(encoding: Encoding, args: struct { + prefix: Instruction.Prefix = .none, + op1: Instruction.Operand = .none, + op2: Instruction.Operand = .none, + op3: Instruction.Operand = .none, + op4: Instruction.Operand = .none, +}) Instruction { + return .{ .encoding = encoding, .prefix = args.prefix, .ops = .{ + args.op1, + args.op2, + args.op3, + args.op4, + } }; +} + +const Prefixes = struct { + legacy: LegacyPrefixes = .{}, + rex: Rex = .{}, + // TODO add support for VEX prefix +}; + +fn parsePrefixes(dis: *Disassembler) !Prefixes { + const rex_prefix_mask: u4 = 0b0100; + var reader: std.Io.Reader = .fixed(dis.code[dis.pos..]); + + var res: Prefixes = .{}; + + while (true) { + const next_byte = try reader.takeByte(); + dis.pos += 1; + + switch (next_byte) { + 0xf0, 0xf2, 0xf3, 0x2e, 0x36, 0x26, 0x64, 0x65, 0x3e, 0x66, 0x67 => { + // Legacy prefix + if (res.rex.present) return error.LegacyPrefixAfterRex; + switch (next_byte) { + 0xf0 => res.legacy.prefix_f0 = true, + 0xf2 => res.legacy.prefix_f2 = true, + 0xf3 => res.legacy.prefix_f3 = true, + 0x2e => res.legacy.prefix_2e = true, + 0x36 => res.legacy.prefix_36 = true, + 0x26 => res.legacy.prefix_26 = true, + 0x64 => res.legacy.prefix_64 = true, + 0x65 => res.legacy.prefix_65 = true, + 0x3e => res.legacy.prefix_3e = true, + 0x66 => res.legacy.prefix_66 = true, + 0x67 => res.legacy.prefix_67 = true, + else => unreachable, + } + }, + else => { + if (rex_prefix_mask == @as(u4, @truncate(next_byte >> 4))) { + // REX prefix + res.rex.w = next_byte & 0b1000 != 0; + res.rex.r = next_byte & 0b100 != 0; + res.rex.x = next_byte & 0b10 != 0; + res.rex.b = next_byte & 0b1 != 0; + res.rex.present = true; + continue; + } + + // TODO VEX prefix + + dis.pos -= 1; + break; + }, + } + } + + return res; +} + +fn parseEncoding(dis: *Disassembler, prefixes: Prefixes) !?Encoding { + const o_mask: u8 = 0b1111_1000; + + var opcode: [3]u8 = .{ 0, 0, 0 }; + var reader: std.Io.Reader = .fixed(dis.code[dis.pos..]); + + comptime var opc_count = 0; + inline while (opc_count < 3) : (opc_count += 1) { + const byte = try reader.takeByte(); + opcode[opc_count] = byte; + dis.pos += 1; + + if (byte == 0x0f) { + // Multi-byte opcode + } else if (opc_count > 0) { + // Multi-byte opcode + if (Encoding.findByOpcode(opcode[0 .. opc_count + 1], .{ + .legacy = prefixes.legacy, + .rex = prefixes.rex, + }, null)) |mnemonic| { + return mnemonic; + } + } else { + // Single-byte opcode + if (Encoding.findByOpcode(opcode[0..1], .{ + .legacy = prefixes.legacy, + .rex = prefixes.rex, + }, null)) |mnemonic| { + return mnemonic; + } else { + // Try O* encoding + return Encoding.findByOpcode(&.{opcode[0] & o_mask}, .{ + .legacy = prefixes.legacy, + .rex = prefixes.rex, + }, null); + } + } + } + return null; +} + +fn parseGpRegister(low_enc: u3, is_extended: bool, rex: Rex, bit_size: u64) Register { + const reg_id: u4 = @as(u4, @intCast(@intFromBool(is_extended))) << 3 | low_enc; + const reg = @as(Register, @enumFromInt(reg_id)).toBitSize(bit_size); + return switch (reg) { + .spl => if (rex.present or rex.isSet()) .spl else .ah, + .dil => if (rex.present or rex.isSet()) .dil else .bh, + .bpl => if (rex.present or rex.isSet()) .bpl else .ch, + .sil => if (rex.present or rex.isSet()) .sil else .dh, + else => reg, + }; +} + +fn parseImm(dis: *Disassembler, kind: Encoding.Op) !Immediate { + var reader: std.Io.Reader = .fixed(dis.code); + reader.seek = dis.pos; + defer dis.pos = reader.seek; + + const imm = switch (kind) { + .imm8s, .rel8 => Immediate.s(try reader.takeInt(i8, .little)), + .imm16s, .rel16 => Immediate.s(try reader.takeInt(i16, .little)), + .imm32s, .rel32 => Immediate.s(try reader.takeInt(i32, .little)), + .imm8 => Immediate.u(try reader.takeInt(u8, .little)), + .imm16 => Immediate.u(try reader.takeInt(u16, .little)), + .imm32 => Immediate.u(try reader.takeInt(u32, .little)), + .imm64 => Immediate.u(try reader.takeInt(u64, .little)), + else => unreachable, + }; + return imm; +} + +fn parseOffset(dis: *Disassembler) !u64 { + var reader: std.Io.Reader = .fixed(dis.code); + reader.seek = dis.pos; + defer dis.pos = reader.seek; + + return reader.takeInt(u64, .little); +} + +const ModRm = packed struct { + mod: u2, + op1: u3, + op2: u3, + + inline fn direct(self: ModRm) bool { + return self.mod == 0b11; + } + + inline fn rip(self: ModRm) bool { + return self.mod == 0 and self.op2 == 0b101; + } + + inline fn sib(self: ModRm) bool { + return !self.direct() and self.op2 == 0b100; + } +}; + +fn parseModRmByte(dis: *Disassembler) !ModRm { + if (dis.code[dis.pos..].len == 0) return error.EndOfStream; + const modrm_byte = dis.code[dis.pos]; + dis.pos += 1; + const mod: u2 = @as(u2, @truncate(modrm_byte >> 6)); + const op1: u3 = @as(u3, @truncate(modrm_byte >> 3)); + const op2: u3 = @as(u3, @truncate(modrm_byte)); + return ModRm{ .mod = mod, .op1 = op1, .op2 = op2 }; +} + +fn segmentRegister(prefixes: LegacyPrefixes) Register { + if (prefixes.prefix_2e) return .cs; + if (prefixes.prefix_36) return .ss; + if (prefixes.prefix_26) return .es; + if (prefixes.prefix_64) return .fs; + if (prefixes.prefix_65) return .gs; + return .ds; +} + +const Sib = packed struct { + scale: u2, + index: u3, + base: u3, + + fn scaleIndex(self: Sib, rex: Rex) ?Memory.ScaleIndex { + if (self.index == 0b100 and !rex.x) return null; + return .{ + .scale = @as(u4, 1) << self.scale, + .index = parseGpRegister(self.index, rex.x, rex, 64), + }; + } + + fn baseReg(self: Sib, modrm: ModRm, prefixes: Prefixes) ?Register { + if (self.base == 0b101 and modrm.mod == 0) { + if (self.scaleIndex(prefixes.rex)) |_| return null; + return segmentRegister(prefixes.legacy); + } + return parseGpRegister(self.base, prefixes.rex.b, prefixes.rex, 64); + } +}; + +fn parseSibByte(dis: *Disassembler) !Sib { + if (dis.code[dis.pos..].len == 0) return error.EndOfStream; + const sib_byte = dis.code[dis.pos]; + dis.pos += 1; + const scale: u2 = @as(u2, @truncate(sib_byte >> 6)); + const index: u3 = @as(u3, @truncate(sib_byte >> 3)); + const base: u3 = @as(u3, @truncate(sib_byte)); + return Sib{ .scale = scale, .index = index, .base = base }; +} + +fn parseDisplacement(dis: *Disassembler, modrm: ModRm, sib: ?Sib) !i32 { + var reader: std.Io.Reader = .fixed(dis.code); + reader.seek = dis.pos; + defer dis.pos = reader.seek; + + const disp = disp: { + if (sib) |info| { + if (info.base == 0b101 and modrm.mod == 0) { + break :disp try reader.takeInt(i32, .little); + } + } + if (modrm.rip()) { + break :disp try reader.takeInt(i32, .little); + } + break :disp switch (modrm.mod) { + 0b00 => 0, + 0b01 => try reader.takeInt(i8, .little), + 0b10 => try reader.takeInt(i32, .little), + 0b11 => unreachable, + }; + }; + return disp; +} diff --git a/src/codegen/x86_64/Emit.zig b/src/codegen/x86_64/Emit.zig new file mode 100644 index 0000000000000000000000000000000000000000..c2b38d8e6d45b3439d2f8bb6cf9e518ac17784cc --- /dev/null +++ b/src/codegen/x86_64/Emit.zig @@ -0,0 +1,982 @@ +//! This file contains the functionality for emitting x86_64 MIR as machine code + +lower: Lower, +bin_file: *link.File, +pt: Zcu.PerThread, +pic: bool, +atom_index: u32, +debug_output: link.File.DebugInfoOutput, +w: *std.Io.Writer, + +prev_di_loc: Loc, +/// Relative to the beginning of `code`. +prev_di_pc: usize, + +code_offset_mapping: std.ArrayListUnmanaged(u32), +relocs: std.ArrayListUnmanaged(Reloc), +table_relocs: std.ArrayListUnmanaged(TableReloc), + +pub const Error = Lower.Error || error{ + EmitFail, + NotFile, +} || std.posix.MMapError || std.posix.MRemapError || link.File.UpdateDebugInfoError; + +pub fn emitMir(emit: *Emit) Error!void { + const comp = emit.bin_file.comp; + const gpa = comp.gpa; + try emit.code_offset_mapping.resize(gpa, emit.lower.mir.instructions.len); + emit.relocs.clearRetainingCapacity(); + emit.table_relocs.clearRetainingCapacity(); + var local_index: usize = 0; + for (0..emit.lower.mir.instructions.len) |mir_i| { + const mir_index: Mir.Inst.Index = @intCast(mir_i); + emit.code_offset_mapping.items[mir_index] = @intCast(emit.w.end); + const lowered = try emit.lower.lowerMir(mir_index); + var lowered_relocs = lowered.relocs; + lowered_inst: for (lowered.insts, 0..) |lowered_inst, lowered_index| { + if (lowered_inst.prefix == .directive) { + const start_offset: u32 = @intCast(emit.w.end); + switch (emit.debug_output) { + .dwarf => |dwarf| switch (lowered_inst.encoding.mnemonic) { + .@".cfi_def_cfa" => try dwarf.genDebugFrame(start_offset, .{ .def_cfa = .{ + .reg = lowered_inst.ops[0].reg.dwarfNum(), + .off = lowered_inst.ops[1].imm.signed, + } }), + .@".cfi_def_cfa_register" => try dwarf.genDebugFrame(start_offset, .{ + .def_cfa_register = lowered_inst.ops[0].reg.dwarfNum(), + }), + .@".cfi_def_cfa_offset" => try dwarf.genDebugFrame(start_offset, .{ + .def_cfa_offset = lowered_inst.ops[0].imm.signed, + }), + .@".cfi_adjust_cfa_offset" => try dwarf.genDebugFrame(start_offset, .{ + .adjust_cfa_offset = lowered_inst.ops[0].imm.signed, + }), + .@".cfi_offset" => try dwarf.genDebugFrame(start_offset, .{ .offset = .{ + .reg = lowered_inst.ops[0].reg.dwarfNum(), + .off = lowered_inst.ops[1].imm.signed, + } }), + .@".cfi_val_offset" => try dwarf.genDebugFrame(start_offset, .{ .val_offset = .{ + .reg = lowered_inst.ops[0].reg.dwarfNum(), + .off = lowered_inst.ops[1].imm.signed, + } }), + .@".cfi_rel_offset" => try dwarf.genDebugFrame(start_offset, .{ .rel_offset = .{ + .reg = lowered_inst.ops[0].reg.dwarfNum(), + .off = lowered_inst.ops[1].imm.signed, + } }), + .@".cfi_register" => try dwarf.genDebugFrame(start_offset, .{ .register = .{ + lowered_inst.ops[0].reg.dwarfNum(), + lowered_inst.ops[1].reg.dwarfNum(), + } }), + .@".cfi_restore" => try dwarf.genDebugFrame(start_offset, .{ + .restore = lowered_inst.ops[0].reg.dwarfNum(), + }), + .@".cfi_undefined" => try dwarf.genDebugFrame(start_offset, .{ + .undefined = lowered_inst.ops[0].reg.dwarfNum(), + }), + .@".cfi_same_value" => try dwarf.genDebugFrame(start_offset, .{ + .same_value = lowered_inst.ops[0].reg.dwarfNum(), + }), + .@".cfi_remember_state" => try dwarf.genDebugFrame(start_offset, .remember_state), + .@".cfi_restore_state" => try dwarf.genDebugFrame(start_offset, .restore_state), + .@".cfi_escape" => try dwarf.genDebugFrame(start_offset, .{ + .escape = lowered_inst.ops[0].bytes, + }), + else => unreachable, + }, + .none => {}, + } + continue; + } + var reloc_info_buf: [2]RelocInfo = undefined; + var reloc_info_index: usize = 0; + while (lowered_relocs.len > 0 and + lowered_relocs[0].lowered_inst_index == lowered_index) : ({ + lowered_relocs = lowered_relocs[1..]; + reloc_info_index += 1; + }) reloc_info_buf[reloc_info_index] = .{ + .op_index = lowered_relocs[0].op_index, + .off = lowered_relocs[0].off, + .target = target: switch (lowered_relocs[0].target) { + .inst => |inst| .{ .index = inst, .is_extern = false, .type = .inst }, + .table => .{ .index = undefined, .is_extern = false, .type = .table }, + .nav => |nav| { + const sym_index = switch (try codegen.genNavRef( + emit.bin_file, + emit.pt, + emit.lower.src_loc, + nav, + emit.lower.target, + )) { + .sym_index => |sym_index| sym_index, + .fail => |em| { + assert(emit.lower.err_msg == null); + emit.lower.err_msg = em; + return error.EmitFail; + }, + }; + const ip = &emit.pt.zcu.intern_pool; + break :target switch (ip.getNav(nav).status) { + .unresolved => unreachable, + .type_resolved => |type_resolved| .{ + .index = sym_index, + .is_extern = false, + .type = if (type_resolved.is_threadlocal and comp.config.any_non_single_threaded) .tlv else .symbol, + }, + .fully_resolved => |fully_resolved| switch (ip.indexToKey(fully_resolved.val)) { + .@"extern" => |@"extern"| .{ + .index = sym_index, + .is_extern = switch (@"extern".visibility) { + .default => true, + .hidden, .protected => false, + }, + .type = if (@"extern".is_threadlocal and comp.config.any_non_single_threaded) .tlv else .symbol, + .force_pcrel_direct = switch (@"extern".relocation) { + .any => false, + .pcrel => true, + }, + }, + .variable => |variable| .{ + .index = sym_index, + .is_extern = false, + .type = if (variable.is_threadlocal and comp.config.any_non_single_threaded) .tlv else .symbol, + }, + else => .{ .index = sym_index, .is_extern = false, .type = .symbol }, + }, + }; + }, + .uav => |uav| .{ + .index = switch (try emit.bin_file.lowerUav( + emit.pt, + uav.val, + Type.fromInterned(uav.orig_ty).ptrAlignment(emit.pt.zcu), + emit.lower.src_loc, + )) { + .sym_index => |sym_index| sym_index, + .fail => |em| { + assert(emit.lower.err_msg == null); + emit.lower.err_msg = em; + return error.EmitFail; + }, + }, + .is_extern = false, + .type = .symbol, + }, + .lazy_sym => |lazy_sym| .{ + .index = if (emit.bin_file.cast(.elf)) |elf_file| + elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, emit.pt, lazy_sym) catch |err| + return emit.fail("{s} creating lazy symbol", .{@errorName(err)}) + else if (emit.bin_file.cast(.elf2)) |elf| + @intFromEnum(try elf.lazySymbol(lazy_sym)) + else if (emit.bin_file.cast(.macho)) |macho_file| + macho_file.getZigObject().?.getOrCreateMetadataForLazySymbol(macho_file, emit.pt, lazy_sym) catch |err| + return emit.fail("{s} creating lazy symbol", .{@errorName(err)}) + else if (emit.bin_file.cast(.coff)) |coff_file| + if (coff_file.getOrCreateAtomForLazySymbol(emit.pt, lazy_sym)) |atom| + coff_file.getAtom(atom).getSymbolIndex().? + else |err| + return emit.fail("{s} creating lazy symbol", .{@errorName(err)}) + else + return emit.fail("lazy symbols unimplemented for {s}", .{@tagName(emit.bin_file.tag)}), + .is_extern = false, + .type = .symbol, + }, + .extern_func => |extern_func| .{ + .index = if (emit.bin_file.cast(.elf)) |elf_file| + try elf_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, null) + else if (emit.bin_file.cast(.elf2)) |elf| @intFromEnum(try elf.globalSymbol(.{ + .name = extern_func.toSlice(&emit.lower.mir).?, + .type = .FUNC, + })) else if (emit.bin_file.cast(.macho)) |macho_file| + try macho_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, null) + else if (emit.bin_file.cast(.coff)) |coff_file| + try coff_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, "compiler_rt") + else + return emit.fail("external symbol unimplemented for {s}", .{@tagName(emit.bin_file.tag)}), + .is_extern = true, + .type = .symbol, + }, + }, + }; + const reloc_info = reloc_info_buf[0..reloc_info_index]; + for (reloc_info) |*reloc| switch (reloc.target.type) { + .inst, .table => {}, + .symbol => { + switch (lowered_inst.encoding.mnemonic) { + .call => { + reloc.target.type = .branch; + if (emit.bin_file.cast(.coff)) |_| try emit.encodeInst(try .new(.none, .call, &.{ + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info) else try emit.encodeInst(lowered_inst, reloc_info); + continue :lowered_inst; + }, + else => {}, + } + if (emit.bin_file.cast(.elf) != null or emit.bin_file.cast(.elf2) != null) { + if (!emit.pic) switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .imm = .s(0) }, + }, emit.lower.target), reloc_info), + .mov => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ + .base = .{ .reg = .ds }, + }) }, + }, emit.lower.target), reloc_info), + else => unreachable, + } else if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info), + .mov => { + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info); + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{ + .reg = lowered_inst.ops[0].reg.to64(), + } }) }, + }, emit.lower.target), &.{}); + }, + else => unreachable, + } else switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .lea, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.none, 0) }, + }, emit.lower.target), reloc_info), + .mov => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, 0) }, + }, emit.lower.target), reloc_info), + else => unreachable, + } + } else if (emit.bin_file.cast(.macho)) |_| { + if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info), + .mov => { + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info); + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{ + .reg = lowered_inst.ops[0].reg.to64(), + } }) }, + }, emit.lower.target), &.{}); + }, + else => unreachable, + } else switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .lea, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.none, 0) }, + }, emit.lower.target), reloc_info), + .mov => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, 0) }, + }, emit.lower.target), reloc_info), + else => unreachable, + } + } else if (emit.bin_file.cast(.coff)) |_| { + if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info), + .mov => { + const dst_reg = lowered_inst.ops[0].reg.to64(); + try emit.encodeInst(try .new(.none, .mov, &.{ + .{ .reg = dst_reg }, + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info); + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{ + .reg = dst_reg, + } }) }, + }, emit.lower.target), &.{}); + }, + else => unreachable, + } else switch (lowered_inst.encoding.mnemonic) { + .lea => try emit.encodeInst(try .new(.none, .lea, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(.none, 0) }, + }, emit.lower.target), reloc_info), + .mov => try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initRip(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, 0) }, + }, emit.lower.target), reloc_info), + else => unreachable, + } + } else return emit.fail("TODO implement relocs for {s}", .{ + @tagName(emit.bin_file.tag), + }); + continue :lowered_inst; + }, + .branch, .tls => unreachable, + .tlv => { + if (emit.bin_file.cast(.elf) != null or emit.bin_file.cast(.elf2) != null) { + // TODO handle extern TLS vars, i.e., emit GD model + if (emit.pic) switch (lowered_inst.encoding.mnemonic) { + .lea, .mov => { + // Here, we currently assume local dynamic TLS vars, and so + // we emit LD model. + try emit.encodeInst(try .new(.none, .lea, &.{ + .{ .reg = .rdi }, + .{ .mem = .initRip(.none, 0) }, + }, emit.lower.target), &.{.{ + .op_index = 1, + .target = .{ + .index = reloc.target.index, + .is_extern = false, + .type = .tls, + }, + }}); + try emit.encodeInst(try .new(.none, .call, &.{ + .{ .imm = .s(0) }, + }, emit.lower.target), &.{.{ + .op_index = 0, + .target = .{ + .index = if (emit.bin_file.cast(.elf)) |elf_file| + try elf_file.getGlobalSymbol("__tls_get_addr", null) + else if (emit.bin_file.cast(.elf2)) |elf| @intFromEnum(try elf.globalSymbol(.{ + .name = "__tls_get_addr", + .type = .FUNC, + })) else unreachable, + .is_extern = true, + .type = .branch, + }, + }}); + try emit.encodeInst(try .new(.none, lowered_inst.encoding.mnemonic, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(.none, .{ + .base = .{ .reg = .rax }, + .disp = std.math.minInt(i32), + }) }, + }, emit.lower.target), reloc_info); + }, + else => unreachable, + } else switch (lowered_inst.encoding.mnemonic) { + .lea, .mov => { + // Since we are linking statically, we emit LE model directly. + try emit.encodeInst(try .new(.none, .mov, &.{ + .{ .reg = .rax }, + .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .fs } }) }, + }, emit.lower.target), &.{}); + try emit.encodeInst(try .new(.none, lowered_inst.encoding.mnemonic, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(.none, .{ + .base = .{ .reg = .rax }, + .disp = std.math.minInt(i32), + }) }, + }, emit.lower.target), reloc_info); + }, + else => unreachable, + } + } else if (emit.bin_file.cast(.macho)) |_| switch (lowered_inst.encoding.mnemonic) { + .lea => { + try emit.encodeInst(try .new(.none, .mov, &.{ + .{ .reg = .rdi }, + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info); + try emit.encodeInst(try .new(.none, .call, &.{ + .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .rdi } }) }, + }, emit.lower.target), &.{}); + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .reg = .rax }, + }, emit.lower.target), &.{}); + }, + .mov => { + try emit.encodeInst(try .new(.none, .mov, &.{ + .{ .reg = .rdi }, + .{ .mem = .initRip(.ptr, 0) }, + }, emit.lower.target), reloc_info); + try emit.encodeInst(try .new(.none, .call, &.{ + .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .rdi } }) }, + }, emit.lower.target), &.{}); + try emit.encodeInst(try .new(.none, .mov, &.{ + lowered_inst.ops[0], + .{ .mem = .initSib(.qword, .{ .base = .{ .reg = .rax } }) }, + }, emit.lower.target), &.{}); + }, + else => unreachable, + } else return emit.fail("TODO implement relocs for {s}", .{ + @tagName(emit.bin_file.tag), + }); + continue :lowered_inst; + }, + }; + try emit.encodeInst(lowered_inst, reloc_info); + } + assert(lowered_relocs.len == 0); + + if (lowered.insts.len == 0) { + const mir_inst = emit.lower.mir.instructions.get(mir_index); + switch (mir_inst.tag) { + else => unreachable, + .pseudo => switch (mir_inst.ops) { + else => unreachable, + .pseudo_dbg_prologue_end_none => switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.setPrologueEnd(), + .none => {}, + }, + .pseudo_dbg_line_stmt_line_column => try emit.dbgAdvancePCAndLine(.{ + .line = mir_inst.data.line_column.line, + .column = mir_inst.data.line_column.column, + .is_stmt = true, + }), + .pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(.{ + .line = mir_inst.data.line_column.line, + .column = mir_inst.data.line_column.column, + .is_stmt = false, + }), + .pseudo_dbg_epilogue_begin_none => switch (emit.debug_output) { + .dwarf => |dwarf| { + try dwarf.setEpilogueBegin(); + log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{ + emit.prev_di_loc.line, emit.prev_di_loc.column, + }); + try emit.dbgAdvancePCAndLine(emit.prev_di_loc); + }, + .none => {}, + }, + .pseudo_dbg_enter_block_none => switch (emit.debug_output) { + .dwarf => |dwarf| { + log.debug("mirDbgEnterBlock (line={d}, col={d})", .{ + emit.prev_di_loc.line, emit.prev_di_loc.column, + }); + try dwarf.enterBlock(emit.w.end); + }, + .none => {}, + }, + .pseudo_dbg_leave_block_none => switch (emit.debug_output) { + .dwarf => |dwarf| { + log.debug("mirDbgLeaveBlock (line={d}, col={d})", .{ + emit.prev_di_loc.line, emit.prev_di_loc.column, + }); + try dwarf.leaveBlock(emit.w.end); + }, + .none => {}, + }, + .pseudo_dbg_enter_inline_func => switch (emit.debug_output) { + .dwarf => |dwarf| { + log.debug("mirDbgEnterInline (line={d}, col={d})", .{ + emit.prev_di_loc.line, emit.prev_di_loc.column, + }); + try dwarf.enterInlineFunc(mir_inst.data.ip_index, emit.w.end, emit.prev_di_loc.line, emit.prev_di_loc.column); + }, + .none => {}, + }, + .pseudo_dbg_leave_inline_func => switch (emit.debug_output) { + .dwarf => |dwarf| { + log.debug("mirDbgLeaveInline (line={d}, col={d})", .{ + emit.prev_di_loc.line, emit.prev_di_loc.column, + }); + try dwarf.leaveInlineFunc(mir_inst.data.ip_index, emit.w.end); + }, + .none => {}, + }, + .pseudo_dbg_arg_none, + .pseudo_dbg_arg_i_s, + .pseudo_dbg_arg_i_u, + .pseudo_dbg_arg_i_64, + .pseudo_dbg_arg_ro, + .pseudo_dbg_arg_fa, + .pseudo_dbg_arg_m, + .pseudo_dbg_var_none, + .pseudo_dbg_var_i_s, + .pseudo_dbg_var_i_u, + .pseudo_dbg_var_i_64, + .pseudo_dbg_var_ro, + .pseudo_dbg_var_fa, + .pseudo_dbg_var_m, + => switch (emit.debug_output) { + .dwarf => |dwarf| { + var loc_buf: [2]link.File.Dwarf.Loc = undefined; + const loc: link.File.Dwarf.Loc = loc: switch (mir_inst.ops) { + else => unreachable, + .pseudo_dbg_arg_none, .pseudo_dbg_var_none => .empty, + .pseudo_dbg_arg_i_s, + .pseudo_dbg_arg_i_u, + .pseudo_dbg_var_i_s, + .pseudo_dbg_var_i_u, + => .{ .stack_value = stack_value: { + loc_buf[0] = switch (emit.lower.imm(mir_inst.ops, mir_inst.data.i.i)) { + .signed => |s| .{ .consts = s }, + .unsigned => |u| .{ .constu = u }, + }; + break :stack_value &loc_buf[0]; + } }, + .pseudo_dbg_arg_i_64, .pseudo_dbg_var_i_64 => .{ .stack_value = stack_value: { + loc_buf[0] = .{ .constu = mir_inst.data.i64 }; + break :stack_value &loc_buf[0]; + } }, + .pseudo_dbg_arg_fa, .pseudo_dbg_var_fa => { + const reg_off = emit.lower.mir.resolveFrameAddr(mir_inst.data.fa); + break :loc .{ .plus = .{ + reg: { + loc_buf[0] = .{ .breg = reg_off.reg.dwarfNum() }; + break :reg &loc_buf[0]; + }, + off: { + loc_buf[1] = .{ .consts = reg_off.off }; + break :off &loc_buf[1]; + }, + } }; + }, + .pseudo_dbg_arg_m, .pseudo_dbg_var_m => { + const mem = emit.lower.mir.resolveMemoryExtra(mir_inst.data.x.payload).decode(); + break :loc .{ .plus = .{ + base: { + loc_buf[0] = switch (mem.base()) { + .none => .{ .constu = 0 }, + .reg => |reg| .{ .breg = reg.dwarfNum() }, + .frame, .table, .rip_inst => unreachable, + .nav => |nav| .{ .addr_reloc = switch (codegen.genNavRef( + emit.bin_file, + emit.pt, + emit.lower.src_loc, + nav, + emit.lower.target, + ) catch |err| switch (err) { + error.CodegenFail, + => return emit.fail("unable to codegen: {s}", .{@errorName(err)}), + else => |e| return e, + }) { + .sym_index => |sym_index| sym_index, + .fail => |em| { + assert(emit.lower.err_msg == null); + emit.lower.err_msg = em; + return error.EmitFail; + }, + } }, + .uav => |uav| .{ .addr_reloc = switch (try emit.bin_file.lowerUav( + emit.pt, + uav.val, + Type.fromInterned(uav.orig_ty).ptrAlignment(emit.pt.zcu), + emit.lower.src_loc, + )) { + .sym_index => |sym_index| sym_index, + .fail => |em| { + assert(emit.lower.err_msg == null); + emit.lower.err_msg = em; + return error.EmitFail; + }, + } }, + .lazy_sym, .extern_func => unreachable, + }; + break :base &loc_buf[0]; + }, + disp: { + loc_buf[1] = switch (mem.disp()) { + .signed => |s| .{ .consts = s }, + .unsigned => |u| .{ .constu = u }, + }; + break :disp &loc_buf[1]; + }, + } }; + }, + }; + + const local = &emit.lower.mir.locals[local_index]; + local_index += 1; + try dwarf.genLocalVarDebugInfo( + switch (mir_inst.ops) { + else => unreachable, + .pseudo_dbg_arg_none, + .pseudo_dbg_arg_i_s, + .pseudo_dbg_arg_i_u, + .pseudo_dbg_arg_i_64, + .pseudo_dbg_arg_ro, + .pseudo_dbg_arg_fa, + .pseudo_dbg_arg_m, + .pseudo_dbg_arg_val, + => .arg, + .pseudo_dbg_var_none, + .pseudo_dbg_var_i_s, + .pseudo_dbg_var_i_u, + .pseudo_dbg_var_i_64, + .pseudo_dbg_var_ro, + .pseudo_dbg_var_fa, + .pseudo_dbg_var_m, + .pseudo_dbg_var_val, + => .local_var, + }, + local.name.toSlice(&emit.lower.mir), + .fromInterned(local.type), + loc, + ); + }, + .none => local_index += 1, + }, + .pseudo_dbg_arg_val, .pseudo_dbg_var_val => switch (emit.debug_output) { + .dwarf => |dwarf| { + const local = &emit.lower.mir.locals[local_index]; + local_index += 1; + try dwarf.genLocalConstDebugInfo( + emit.lower.src_loc, + switch (mir_inst.ops) { + else => unreachable, + .pseudo_dbg_arg_val => .comptime_arg, + .pseudo_dbg_var_val => .local_const, + }, + local.name.toSlice(&emit.lower.mir), + .fromInterned(mir_inst.data.ip_index), + ); + }, + .none => local_index += 1, + }, + .pseudo_dbg_var_args_none => switch (emit.debug_output) { + .dwarf => |dwarf| try dwarf.genVarArgsDebugInfo(), + .none => {}, + }, + .pseudo_dead_none => {}, + }, + } + } + } + for (emit.relocs.items) |reloc| { + const target = emit.code_offset_mapping.items[reloc.target]; + const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.inst_offset + reloc.inst_length)) + reloc.target_offset; + const inst_bytes = emit.w.buffered()[reloc.inst_offset..][0..reloc.inst_length]; + switch (reloc.source_length) { + else => unreachable, + inline 1, 4 => |source_length| std.mem.writeInt( + @Type(.{ .int = .{ .signedness = .signed, .bits = @as(u16, 8) * source_length } }), + inst_bytes[reloc.source_offset..][0..source_length], + @intCast(disp), + .little, + ), + } + } + if (emit.lower.mir.table.len > 0) { + const ptr_size = @divExact(emit.lower.target.ptrBitWidth(), 8); + var table_offset = std.mem.alignForward(u32, @intCast(emit.w.end), ptr_size); + if (emit.bin_file.cast(.elf)) |elf_file| { + const zo = elf_file.zigObjectPtr().?; + const atom = zo.symbol(emit.atom_index).atom(elf_file).?; + + for (emit.table_relocs.items) |table_reloc| try atom.addReloc(gpa, .{ + .r_offset = table_reloc.source_offset, + .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"32"), + .r_addend = @as(i64, table_offset) + table_reloc.target_offset, + }, zo); + for (emit.lower.mir.table) |entry| { + try atom.addReloc(gpa, .{ + .r_offset = table_offset, + .r_info = @as(u64, emit.atom_index) << 32 | @intFromEnum(std.elf.R_X86_64.@"64"), + .r_addend = emit.code_offset_mapping.items[entry], + }, zo); + table_offset += ptr_size; + } + try emit.w.splatByteAll(0, table_offset - emit.w.end); + } else if (emit.bin_file.cast(.elf2)) |elf| { + for (emit.table_relocs.items) |table_reloc| try elf.addReloc( + @enumFromInt(emit.atom_index), + table_reloc.source_offset, + @enumFromInt(emit.atom_index), + @as(i64, table_offset) + table_reloc.target_offset, + .{ .x86_64 = .@"32" }, + ); + for (emit.lower.mir.table) |entry| { + try elf.addReloc( + @enumFromInt(emit.atom_index), + table_offset, + @enumFromInt(emit.atom_index), + emit.code_offset_mapping.items[entry], + .{ .x86_64 = .@"64" }, + ); + table_offset += ptr_size; + } + try emit.w.splatByteAll(0, table_offset - emit.w.end); + } else unreachable; + } +} + +pub fn deinit(emit: *Emit) void { + const gpa = emit.bin_file.comp.gpa; + emit.code_offset_mapping.deinit(gpa); + emit.relocs.deinit(gpa); + emit.table_relocs.deinit(gpa); + emit.* = undefined; +} + +const RelocInfo = struct { + op_index: Lower.InstOpIndex, + off: i32 = 0, + target: Target, + + const Target = struct { + index: u32, + is_extern: bool, + type: Target.Type, + force_pcrel_direct: bool = false, + + const Type = enum { inst, table, symbol, branch, tls, tlv }; + }; +}; + +fn encodeInst(emit: *Emit, lowered_inst: Instruction, reloc_info: []const RelocInfo) Error!void { + const comp = emit.bin_file.comp; + const gpa = comp.gpa; + const start_offset: u32 = @intCast(emit.w.end); + lowered_inst.encode(emit.w, .{}) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, + }; + const end_offset: u32 = @intCast(emit.w.end); + for (reloc_info) |reloc| switch (reloc.target.type) { + .inst => { + const inst_length: u4 = @intCast(end_offset - start_offset); + const reloc_offset, const reloc_length = reloc_offset_length: { + var reloc_offset = inst_length; + var op_index: usize = lowered_inst.ops.len; + while (true) { + op_index -= 1; + const op = lowered_inst.encoding.data.ops[op_index]; + if (op == .none) continue; + const is_mem = op.isMemory(); + const enc_length: u4 = if (is_mem) switch (lowered_inst.ops[op_index].mem.sib.base) { + .rip_inst => 4, + else => unreachable, + } else @intCast(std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable); + reloc_offset -= enc_length; + if (op_index == reloc.op_index) break :reloc_offset_length .{ reloc_offset, enc_length }; + assert(!is_mem); + } + }; + try emit.relocs.append(emit.lower.allocator, .{ + .inst_offset = start_offset, + .inst_length = inst_length, + .source_offset = reloc_offset, + .source_length = reloc_length, + .target = reloc.target.index, + .target_offset = reloc.off, + }); + }, + .table => try emit.table_relocs.append(emit.lower.allocator, .{ + .source_offset = end_offset - 4, + .target_offset = reloc.off, + }), + .symbol => if (emit.bin_file.cast(.elf)) |elf_file| { + const zo = elf_file.zigObjectPtr().?; + const atom = zo.symbol(emit.atom_index).atom(elf_file).?; + const r_type: std.elf.R_X86_64 = if (!emit.pic) + .@"32" + else if (reloc.target.is_extern and !reloc.target.force_pcrel_direct) + .GOTPCREL + else + .PC32; + try atom.addReloc(gpa, .{ + .r_offset = end_offset - 4, + .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), + .r_addend = if (emit.pic) reloc.off - 4 else reloc.off, + }, zo); + } else if (emit.bin_file.cast(.macho)) |macho_file| { + const zo = macho_file.getZigObject().?; + const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?; + try atom.addReloc(macho_file, .{ + .tag = .@"extern", + .offset = end_offset - 4, + .target = reloc.target.index, + .addend = reloc.off, + .type = if (reloc.target.is_extern and !reloc.target.force_pcrel_direct) .got_load else .signed, + .meta = .{ + .pcrel = true, + .has_subtractor = false, + .length = 2, + .symbolnum = @intCast(reloc.target.index), + }, + }); + } else if (emit.bin_file.cast(.elf2)) |elf| try elf.addReloc( + @enumFromInt(emit.atom_index), + end_offset - 4, + @enumFromInt(reloc.target.index), + reloc.off, + .{ .x86_64 = .@"32" }, + ) else if (emit.bin_file.cast(.coff)) |coff_file| { + const atom_index = coff_file.getAtomIndexForSymbol( + .{ .sym_index = emit.atom_index, .file = null }, + ).?; + try coff_file.addRelocation(atom_index, .{ + .type = if (reloc.target.is_extern) .got else .direct, + .target = if (reloc.target.is_extern) + coff_file.getGlobalByIndex(reloc.target.index) + else + .{ .sym_index = reloc.target.index, .file = null }, + .offset = end_offset - 4, + .addend = @intCast(reloc.off), + .pcrel = true, + .length = 2, + }); + } else unreachable, + .branch => if (emit.bin_file.cast(.elf)) |elf_file| { + const zo = elf_file.zigObjectPtr().?; + const atom = zo.symbol(emit.atom_index).atom(elf_file).?; + const r_type: std.elf.R_X86_64 = .PLT32; + try atom.addReloc(gpa, .{ + .r_offset = end_offset - 4, + .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), + .r_addend = reloc.off - 4, + }, zo); + } else if (emit.bin_file.cast(.elf2)) |elf| try elf.addReloc( + @enumFromInt(emit.atom_index), + end_offset - 4, + @enumFromInt(reloc.target.index), + reloc.off - 4, + .{ .x86_64 = .PC32 }, + ) else if (emit.bin_file.cast(.macho)) |macho_file| { + const zo = macho_file.getZigObject().?; + const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?; + try atom.addReloc(macho_file, .{ + .tag = .@"extern", + .offset = end_offset - 4, + .target = reloc.target.index, + .addend = reloc.off, + .type = .branch, + .meta = .{ + .pcrel = true, + .has_subtractor = false, + .length = 2, + .symbolnum = @intCast(reloc.target.index), + }, + }); + } else if (emit.bin_file.cast(.coff)) |coff_file| { + const atom_index = coff_file.getAtomIndexForSymbol( + .{ .sym_index = emit.atom_index, .file = null }, + ).?; + try coff_file.addRelocation(atom_index, .{ + .type = if (reloc.target.is_extern) .import else .got, + .target = if (reloc.target.is_extern) + coff_file.getGlobalByIndex(reloc.target.index) + else + .{ .sym_index = reloc.target.index, .file = null }, + .offset = end_offset - 4, + .addend = @intCast(reloc.off), + .pcrel = true, + .length = 2, + }); + } else return emit.fail("TODO implement {s} reloc for {s}", .{ + @tagName(reloc.target.type), @tagName(emit.bin_file.tag), + }), + .tls => if (emit.bin_file.cast(.elf)) |elf_file| { + const zo = elf_file.zigObjectPtr().?; + const atom = zo.symbol(emit.atom_index).atom(elf_file).?; + const r_type: std.elf.R_X86_64 = if (emit.pic) .TLSLD else unreachable; + try atom.addReloc(gpa, .{ + .r_offset = end_offset - 4, + .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), + .r_addend = reloc.off - 4, + }, zo); + } else return emit.fail("TODO implement {s} reloc for {s}", .{ + @tagName(reloc.target.type), @tagName(emit.bin_file.tag), + }), + .tlv => if (emit.bin_file.cast(.elf)) |elf_file| { + const zo = elf_file.zigObjectPtr().?; + const atom = zo.symbol(emit.atom_index).atom(elf_file).?; + const r_type: std.elf.R_X86_64 = if (emit.pic) .DTPOFF32 else .TPOFF32; + try atom.addReloc(gpa, .{ + .r_offset = end_offset - 4, + .r_info = @as(u64, reloc.target.index) << 32 | @intFromEnum(r_type), + .r_addend = reloc.off, + }, zo); + } else if (emit.bin_file.cast(.elf2)) |elf| try elf.addReloc( + @enumFromInt(emit.atom_index), + end_offset - 4, + @enumFromInt(reloc.target.index), + reloc.off, + .{ .x86_64 = .TPOFF32 }, + ) else if (emit.bin_file.cast(.macho)) |macho_file| { + const zo = macho_file.getZigObject().?; + const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?; + try atom.addReloc(macho_file, .{ + .tag = .@"extern", + .offset = end_offset - 4, + .target = reloc.target.index, + .addend = reloc.off, + .type = .tlv, + .meta = .{ + .pcrel = true, + .has_subtractor = false, + .length = 2, + .symbolnum = @intCast(reloc.target.index), + }, + }); + } else return emit.fail("TODO implement {s} reloc for {s}", .{ + @tagName(reloc.target.type), @tagName(emit.bin_file.tag), + }), + }; +} + +fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error { + return switch (emit.lower.fail(format, args)) { + error.LowerFail => error.EmitFail, + else => |e| e, + }; +} + +const Reloc = struct { + /// Offset of the instruction. + inst_offset: u32, + /// Length of the instruction. + inst_length: u4, + /// Offset of the relocation within the instruction. + source_offset: u4, + /// Length of the relocation. + source_length: u4, + /// Target of the relocation. + target: Mir.Inst.Index, + /// Offset from the target. + target_offset: i32, +}; + +const TableReloc = struct { + /// Offset of the relocation. + source_offset: u32, + /// Offset from the start of the table. + target_offset: i32, +}; + +const Loc = struct { + line: u32, + column: u32, + is_stmt: bool, +}; + +fn dbgAdvancePCAndLine(emit: *Emit, loc: Loc) Error!void { + const delta_line = @as(i33, loc.line) - @as(i33, emit.prev_di_loc.line); + const delta_pc: usize = emit.w.end - emit.prev_di_pc; + log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line }); + switch (emit.debug_output) { + .dwarf => |dwarf| { + if (loc.is_stmt != emit.prev_di_loc.is_stmt) try dwarf.negateStmt(); + if (loc.column != emit.prev_di_loc.column) try dwarf.setColumn(loc.column); + try dwarf.advancePCAndLine(delta_line, delta_pc); + emit.prev_di_loc = loc; + emit.prev_di_pc = emit.w.end; + }, + .none => {}, + } +} + +const assert = std.debug.assert; +const bits = @import("bits.zig"); +const codegen = @import("../../codegen.zig"); +const Emit = @This(); +const encoder = @import("encoder.zig"); +const Instruction = encoder.Instruction; +const InternPool = @import("../../InternPool.zig"); +const link = @import("../../link.zig"); +const log = std.log.scoped(.emit); +const Lower = @import("Lower.zig"); +const Mir = @import("Mir.zig"); +const std = @import("std"); +const Type = @import("../../Type.zig"); +const Zcu = @import("../../Zcu.zig"); diff --git a/src/codegen/x86_64/Encoding.zig b/src/codegen/x86_64/Encoding.zig new file mode 100644 index 0000000000000000000000000000000000000000..3b41e0b07afdc16b5136aeb0ee4d89ab784a2572 --- /dev/null +++ b/src/codegen/x86_64/Encoding.zig @@ -0,0 +1,1072 @@ +const Encoding = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const math = std.math; + +const bits = @import("bits.zig"); +const encoder = @import("encoder.zig"); +const Instruction = encoder.Instruction; +const Operand = Instruction.Operand; +const Prefix = Instruction.Prefix; +const Register = bits.Register; +const Rex = encoder.Rex; +const LegacyPrefixes = encoder.LegacyPrefixes; + +mnemonic: Mnemonic, +data: Data, + +const Data = struct { + op_en: OpEn, + ops: [4]Op, + opc_len: u3, + opc: [7]u8, + modrm_ext: u3, + mode: Mode, + feature: Feature, +}; + +pub fn findByMnemonic( + prefix: Instruction.Prefix, + mnemonic: Mnemonic, + ops: []const Instruction.Operand, + target: *const std.Target, +) !?Encoding { + var input_ops: [4]Op = @splat(.none); + for (input_ops[0..ops.len], ops) |*input_op, op| input_op.* = Op.fromOperand(op, target); + + const rex_required = for (ops) |op| switch (op) { + .reg => |r| switch (r) { + .spl, .bpl, .sil, .dil => break true, + else => {}, + }, + else => {}, + } else false; + const rex_invalid = for (ops) |op| switch (op) { + .reg => |r| switch (r) { + .ah, .bh, .ch, .dh => break true, + else => {}, + }, + else => {}, + } else false; + const rex_extended = for (ops) |op| { + if (op.baseExtEnc() != 0b00 or op.indexExtEnc() != 0b00) break true; + } else false; + + if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode; + + var shortest_enc: ?Encoding = null; + var shortest_len: ?usize = null; + next: for (mnemonic_to_encodings_map[@intFromEnum(mnemonic)]) |data| { + if (!switch (data.feature) { + .none => true, + .@"32bit" => switch (target.cpu.arch) { + else => unreachable, + .x86 => true, + .x86_64 => false, + }, + .@"64bit" => switch (target.cpu.arch) { + else => unreachable, + .x86 => false, + .x86_64 => true, + }, + inline .@"invpcid 32bit", .@"rdpid 32bit" => |tag| switch (target.cpu.arch) { + else => unreachable, + .x86 => target.cpu.has( + .x86, + @field(std.Target.x86.Feature, @tagName(tag)[0 .. @tagName(tag).len - " 32bit".len]), + ), + .x86_64 => false, + }, + inline .@"invpcid 64bit", .@"rdpid 64bit", .@"prefetchi 64bit" => |tag| switch (target.cpu.arch) { + else => unreachable, + .x86 => false, + .x86_64 => target.cpu.has( + .x86, + @field(std.Target.x86.Feature, @tagName(tag)[0 .. @tagName(tag).len - " 64bit".len]), + ), + }, + .prefetch => target.cpu.hasAny(.x86, &.{ .sse, .prfchw, .prefetchi, .prefetchwt1 }), + inline else => |tag| has_features: { + comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' '); + comptime var features: []const std.Target.x86.Feature = &.{}; + inline while (comptime feature_it.next()) |feature| features = features ++ .{@field(std.Target.x86.Feature, feature)}; + break :has_features target.cpu.hasAll(.x86, features); + }, + }) continue; + + switch (data.mode) { + .none, .short => if (rex_required) continue, + .rex, .rex_short => if (!rex_required) continue, + else => {}, + } + for (input_ops, data.ops) |input_op, data_op| if (!input_op.isSubset(data_op)) continue :next; + + const enc: Encoding = .{ .mnemonic = mnemonic, .data = data }; + if (shortest_enc) |previous_shortest_enc| { + const len = estimateInstructionLength(prefix, enc, ops); + const previous_shortest_len = shortest_len orelse + estimateInstructionLength(prefix, previous_shortest_enc, ops); + if (len < previous_shortest_len) { + shortest_enc = enc; + shortest_len = len; + } else shortest_len = previous_shortest_len; + } else shortest_enc = enc; + } + return shortest_enc; +} + +/// Returns first matching encoding by opcode. +pub fn findByOpcode(opc: []const u8, prefixes: struct { + legacy: LegacyPrefixes, + rex: Rex, +}, modrm_ext: ?u3) ?Encoding { + for (mnemonic_to_encodings_map, 0..) |encs, mnemonic_int| for (encs) |data| { + const enc = Encoding{ .mnemonic = @as(Mnemonic, @enumFromInt(mnemonic_int)), .data = data }; + if (modrm_ext) |ext| if (ext != data.modrm_ext) continue; + if (!std.mem.eql(u8, opc, enc.opcode())) continue; + if (prefixes.rex.w) { + if (!data.mode.isLong()) continue; + } else if (prefixes.rex.present and !prefixes.rex.isSet()) { + if (!data.mode.isRex()) continue; + } else if (prefixes.legacy.prefix_66) { + if (!data.mode.isShort()) continue; + } else { + if (data.mode.isShort()) continue; + } + return enc; + }; + return null; +} + +pub fn opcode(encoding: *const Encoding) []const u8 { + return encoding.data.opc[0..encoding.data.opc_len]; +} + +pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 { + const prefix = encoding.data.opc[0]; + return switch (prefix) { + 0x66, 0xf2, 0xf3 => prefix, + else => null, + }; +} + +pub fn modRmExt(encoding: Encoding) u3 { + return switch (encoding.data.op_en) { + .ia, .m, .mi, .m1, .mc, .vm, .vmi => encoding.data.modrm_ext, + else => unreachable, + }; +} + +pub fn format(encoding: Encoding, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var opc = encoding.opcode(); + if (encoding.data.mode.isVex()) { + try writer.writeAll("VEX."); + + try writer.writeAll(switch (encoding.data.mode) { + .vex_128_w0, .vex_128_w1, .vex_128_wig => "128", + .vex_256_w0, .vex_256_w1, .vex_256_wig => "256", + .vex_lig_w0, .vex_lig_w1, .vex_lig_wig => "LIG", + .vex_lz_w0, .vex_lz_w1, .vex_lz_wig => "LZ", + else => unreachable, + }); + + switch (opc[0]) { + else => {}, + 0x66, 0xf3, 0xf2 => { + try writer.print(".{X:0>2}", .{opc[0]}); + opc = opc[1..]; + }, + } + + try writer.print(".{X}", .{opc[0 .. opc.len - 1]}); + opc = opc[opc.len - 1 ..]; + + try writer.writeAll(".W"); + try writer.writeAll(switch (encoding.data.mode) { + .vex_128_w0, .vex_256_w0, .vex_lig_w0, .vex_lz_w0 => "0", + .vex_128_w1, .vex_256_w1, .vex_lig_w1, .vex_lz_w1 => "1", + .vex_128_wig, .vex_256_wig, .vex_lig_wig, .vex_lz_wig => "IG", + else => unreachable, + }); + + try writer.writeByte(' '); + } else if (encoding.data.mode.isLong()) try writer.writeAll("REX.W + "); + for (opc) |byte| try writer.print("{x:0>2} ", .{byte}); + + switch (encoding.data.op_en) { + .z, .fd, .td, .i, .zi, .ii, .d => {}, + .o, .zo, .oz, .oi => { + const op = switch (encoding.data.op_en) { + .o, .oz, .oi => encoding.data.ops[0], + .zo => encoding.data.ops[1], + else => unreachable, + }; + const tag = switch (op) { + .r8 => "rb", + .r16 => "rw", + .r32 => "rd", + .r64 => "rd", + else => unreachable, + }; + try writer.print("+{s} ", .{tag}); + }, + .ia, .m, .mi, .m1, .mc, .vm, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}), + .mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr, .rmv => try writer.writeAll("/r "), + } + + switch (encoding.data.op_en) { + .i, .d, .zi, .ii, .ia, .oi, .mi, .rmi, .mri, .vmi, .rvmi => for (0..2) |i| { + const op = switch (i) { + 0 => switch (encoding.data.op_en) { + .i, .ii, .ia, .d => encoding.data.ops[0], + .zi, .oi, .mi => encoding.data.ops[1], + .rmi, .mri, .vmi => encoding.data.ops[2], + .rvmi => encoding.data.ops[3], + else => unreachable, + }, + 1 => switch (encoding.data.op_en) { + .ii => encoding.data.ops[1], + else => break, + }, + else => unreachable, + }; + const tag = switch (op) { + .imm8, .imm8s => "ib", + .imm16, .imm16s => "iw", + .imm32, .imm32s => "id", + .imm64 => "io", + .rel8 => "cb", + .rel16 => "cw", + .rel32 => "cd", + else => unreachable, + }; + try writer.print("{s} ", .{tag}); + }, + .rvmr => try writer.writeAll("/is4 "), + .z, .fd, .td, .o, .zo, .oz, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .vm, .rvm, .mvr, .rmv => {}, + } + + try writer.print("{s} ", .{@tagName(encoding.mnemonic)}); + + for (encoding.data.ops) |op| switch (op) { + .none => break, + else => try writer.print("{s} ", .{@tagName(op)}), + }; + + const op_en = switch (encoding.data.op_en) { + .zi => .i, + else => |op_en| op_en, + }; + try writer.print("{s}", .{@tagName(op_en)}); +} + +pub const Mnemonic = enum { + // Directives + @".cfi_def_cfa", + @".cfi_def_cfa_register", + @".cfi_def_cfa_offset", + @".cfi_adjust_cfa_offset", + @".cfi_offset", + @".cfi_val_offset", + @".cfi_rel_offset", + @".cfi_register", + @".cfi_restore", + @".cfi_undefined", + @".cfi_same_value", + @".cfi_remember_state", + @".cfi_restore_state", + @".cfi_escape", + // zig fmt: off + // General-purpose + aaa, aad, aam, aas, adc, add, @"and", arpl, + bound, bsf, bsr, bswap, bt, btc, btr, bts, + call, cbw, cdq, cdqe, + clac, clc, cld, cldemote, clflush, clflushopt, cli, clts, clui, clrssbsy, clwb, cmc, + cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna, + cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno, + cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz, + cmp, cmps, cmpsb, cmpsd, cmpsq, cmpsw, cmpxchg, cmpxchg8b, cmpxchg16b, + cpuid, cqo, cwd, cwde, + daa, das, dec, div, + endbr32, endbr64, enqcmd, enqcmds, enter, + hlt, hreset, + idiv, imul, in, inc, incsspd, incsspq, ins, insb, insd, insw, + int, int1, int3, into, invd, invlpg, invpcid, iret, iretd, iretq, iretw, + ja, jae, jb, jbe, jc, jcxz, je, jecxz, jg, jge, jl, jle, jmp, jna, jnae, jnb, jnbe, + jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, jrcxz, js, jz, + lahf, lar, lea, leave, lfence, lgdt, lidt, lldt, lmsw, loop, loope, loopne, + lods, lodsb, lodsd, lodsq, lodsw, + lsl, ltr, + mfence, mov, movbe, + movs, movsb, movsd, movsq, movsw, + movsx, movsxd, movzx, mul, + neg, nop, not, + @"or", out, outs, outsb, outsd, outsw, + pause, pop, popf, popfd, popfq, push, pushfq, + rcl, rcr, + rdfsbase, rdgsbase, rdmsr, rdpid, rdpkru, rdpmc, rdrand, rdseed, rdsspd, rdsspq, rdtsc, rdtscp, + ret, rol, ror, rsm, + sahf, sal, sar, sbb, + scas, scasb, scasd, scasq, scasw, + senduipi, serialize, + shl, shld, shr, shrd, + stac, stc, std, sti, str, stui, + sub, swapgs, syscall, sysenter, sysexit, sysret, + seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae, + setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns, + setnz, seto, setp, setpe, setpo, sets, setz, + sfence, sidt, sldt, smsw, + stos, stosb, stosd, stosq, stosw, + @"test", testui, tpause, + ud0, ud1, ud2, uiret, umonitor, umwait, + verr, verw, wrfsbase, wrgsbase, wrmsr, wrpkru, wrssd, wrssq, wrussd, wrussq, + xadd, xchg, xgetbv, xlat, xlatb, xor, + // X87 + f2xm1, fabs, fadd, faddp, fbld, fbstp, fchs, fclex, + fcmovb, fcmovbe, fcmove, fcmovnb, fcmovnbe, fcmovne, fcmovnu, fcmovu, + fcom, fcomi, fcomip, fcomp, fcompp, fcos, + fdecstp, fdiv, fdivp, fdivr, fdivrp, ffree, + fiadd, ficom, ficomp, fidiv, fidivr, fild, fimul, fincstp, finit, + fist, fistp, fisub, fisubr, + fld, fld1, fldcw, fldenv, fldl2e, fldl2t, fldlg2, fldln2, fldpi, fldz, + fmul, fmulp, + fnclex, fninit, fnop, fnsave, fnstcw, fnstenv, fnstsw, + fpatan, fprem, fprem1, fptan, frndint, frstor, + fsave, fscale, fsin, fsincos, fsqrt, + fst, fstcw, fstenv, fstp, fstsw, + fsub, fsubp, fsubr, fsubrp, + ftst, fucom, fucomi, fucomip, fucomp, fucompp, + fwait, fxam, fxch, fxtract, fyl2x, fyl2xp1, wait, + // MMX + emms, movd, movq, + packssdw, packsswb, packuswb, + paddb, paddd, paddsb, paddsw, paddusb, paddusw, paddw, + pand, pandn, por, pxor, + pcmpeqb, pcmpeqd, pcmpeqw, + pcmpgtb, pcmpgtd, pcmpgtw, + pmaddwd, pmulhw, pmullw, + pslld, psllq, psllw, + psrad, psraw, + psrld, psrlq, psrlw, + psubb, psubd, psubsb, psubsw, psubusb, psubusw, psubw, + // SSE + addps, addss, + andnps, andps, + cmpps, cmpss, comiss, + cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si, + divps, divss, + fxrstor, fxrstor64, fxsave, fxsave64, + ldmxcsr, + maxps, maxss, + minps, minss, + movaps, movhlps, movhps, movlhps, movlps, + movmskps, + movss, movups, + mulps, mulss, + orps, + pavgb, pavgw, + pextrw, pinsrw, + pmaxsw, pmaxub, pminsw, pminub, pmovmskb, pmulhuw, + prefetchit0, prefetchit1, prefetchnta, prefetcht0, prefetcht1, prefetcht2, prefetchw, prefetchwt1, + psadbw, pshufw, + shufps, + sqrtps, sqrtss, + stmxcsr, + subps, subss, + ucomiss, unpckhps, unpcklps, + xorps, + // SSE2 + addpd, addsd, + andpd, + andnpd, + cmppd, //cmpsd, + comisd, + cvtdq2pd, cvtdq2ps, cvtpd2dq, cvtpd2pi, cvtpd2ps, cvtpi2pd, + cvtps2dq, cvtps2pd, cvtsd2si, cvtsd2ss, cvtsi2sd, cvtss2sd, + cvttpd2dq, cvttpd2pi, cvttps2dq, cvttsd2si, + divpd, divsd, + gf2p8affineinvqb, gf2p8affineqb, gf2p8mulb, + maxpd, maxsd, + minpd, minsd, + movapd, + movdq2q, movdqa, movdqu, + movhpd, movlpd, + movmskpd, movq2dq, + //movsd, + movupd, + mulpd, mulsd, + orpd, + paddq, pmuludq, + pshufd, pshufhw, pshuflw, + pslldq, psrldq, psubq, + punpckhbw, punpckhdq, punpckhqdq, punpckhwd, + punpcklbw, punpckldq, punpcklqdq, punpcklwd, + shufpd, + sqrtpd, sqrtsd, + subpd, subsd, + ucomisd, unpckhpd, unpcklpd, + xorpd, + // SSE3 + addsubpd, addsubps, + fisttp, + haddpd, haddps, + hsubpd, hsubps, + lddqu, + movddup, movshdup, movsldup, + // SSSE3 + pabsb, pabsd, pabsw, palignr, + phaddw, phaddsw, phaddd, phsubw, phsubsw, phsubd, + pmaddubsw, pmulhrsw, pshufb, + psignb, psignd, psignw, + // SSE4.1 + blendpd, blendps, blendvpd, blendvps, + dppd, dpps, + extractps, + insertps, + packusdw, + pblendvb, pblendw, + pcmpeqq, + pextrb, pextrd, pextrq, + phminposuw, + pinsrb, pinsrd, pinsrq, + pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw, + pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq, + pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq, + pmuldq, pmulld, + ptest, + roundpd, roundps, roundsd, roundss, + // SSE4.2 + crc32, pcmpgtq, + // ABM + lzcnt, popcnt, + // PCLMUL + pclmulqdq, + // AES + aesdec, aesdeclast, aesenc, aesenclast, aesimc, aeskeygenassist, + // SHA + sha1rnds4, sha1nexte, sha1msg1, sha1msg2, sha256msg1, sha256msg2, sha256rnds2, + // AVX + vaddpd, vaddps, vaddsd, vaddss, vaddsubpd, vaddsubps, + vaesdec, vaesdeclast, vaesenc, vaesenclast, vaesimc, vaeskeygenassist, + vandnpd, vandnps, vandpd, vandps, + vblendpd, vblendps, vblendvpd, vblendvps, + vbroadcastf128, vbroadcastsd, vbroadcastss, + vcmppd, vcmpps, vcmpsd, vcmpss, vcomisd, vcomiss, + vcvtdq2pd, vcvtdq2ps, vcvtpd2dq, vcvtpd2ps, + vcvtps2dq, vcvtps2pd, vcvtsd2si, vcvtsd2ss, + vcvtsi2sd, vcvtsi2ss, vcvtss2sd, vcvtss2si, + vcvttpd2dq, vcvttps2dq, vcvttsd2si, vcvttss2si, + vdivpd, vdivps, vdivsd, vdivss, + vdppd, vdpps, + vextractf128, vextractps, + vgf2p8affineinvqb, vgf2p8affineqb, vgf2p8mulb, + vhaddpd, vhaddps, vhsubpd, vhsubps, + vinsertf128, vinsertps, + vlddqu, vldmxcsr, + vmaskmovpd, vmaskmovps, + vmaxpd, vmaxps, vmaxsd, vmaxss, + vminpd, vminps, vminsd, vminss, + vmovapd, vmovaps, + vmovd, + vmovddup, + vmovdqa, vmovdqu, + vmovhlps, vmovhpd, vmovhps, vmovlhps, vmovlpd, vmovlps, + vmovmskpd, vmovmskps, + vmovq, + vmovsd, + vmovshdup, vmovsldup, + vmovss, + vmovupd, vmovups, + vmulpd, vmulps, vmulsd, vmulss, + vorpd, vorps, + vpabsb, vpabsd, vpabsw, + vpackssdw, vpacksswb, vpackusdw, vpackuswb, + vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw, + vpalignr, vpand, vpandn, vpavgb, vpavgw, + vpblendvb, vpblendw, vpclmulqdq, + vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw, + vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw, + vperm2f128, vpermilpd, vpermilps, + vpextrb, vpextrd, vpextrq, vpextrw, + vphaddw, vphaddsw, vphaddd, vphminposuw, vphsubw, vphsubsw, vphsubd, + vpinsrb, vpinsrd, vpinsrq, vpinsrw, + vpmaddubsw, vpmaddwd, + vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw, + vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw, + vpmovmskb, + vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq, + vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq, + vpmuldq, vpmulhrsw, vpmulhuw, vpmulhw, vpmulld, vpmullw, vpmuludq, + vpor, + vpsadbw, vpshufb, vpshufd, vpshufhw, vpshuflw, + vpsignb, vpsignd, vpsignw, + vpslld, vpslldq, vpsllq, vpsllw, + vpsrad, vpsraq, vpsraw, + vpsrld, vpsrldq, vpsrlq, vpsrlw, + vpsubb, vpsubd, vpsubq, vpsubsb, vpsubsw, vpsubusb, vpsubusw, vpsubw, + vptest, + vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd, + vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd, + vpxor, + vroundpd, vroundps, vroundsd, vroundss, + vshufpd, vshufps, + vsqrtpd, vsqrtps, vsqrtsd, vsqrtss, + vstmxcsr, + vsubpd, vsubps, vsubsd, vsubss, + vtestpd, vtestps, + vucomisd, vucomiss, vunpckhpd, vunpckhps, vunpcklpd, vunpcklps, + vxorpd, vxorps, + // BMI + andn, bextr, blsi, blsmsk, blsr, tzcnt, + // BMI2 + bzhi, mulx, pdep, pext, rorx, sarx, shlx, shrx, + // F16C + vcvtph2ps, vcvtps2ph, + // FMA + vfmadd132pd, vfmadd213pd, vfmadd231pd, + vfmadd132ps, vfmadd213ps, vfmadd231ps, + vfmadd132sd, vfmadd213sd, vfmadd231sd, + vfmadd132ss, vfmadd213ss, vfmadd231ss, + // AVX2 + vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw, + vextracti128, vinserti128, vpblendd, + vperm2i128, vpermd, vpermpd, vpermps, vpermq, + vpmaskmovd, vpmaskmovq, + vpsllvd, vpsllvq, vpsravd, vpsrlvd, vpsrlvq, + // ADX + adcx, adox, + // AESKLE + aesdec128kl, aesdec256kl, aesenc128kl, aesenc256kl, encodekey128, encodekey256, loadiwkey, + // AESKLEWIDE_KL + aesdecwide128kl, aesdecwide256kl, aesencwide128kl, aesencwide256kl, + // zig fmt: on +}; + +pub const OpEn = enum { + // zig fmt: off + z, + o, zo, oz, oi, + i, zi, ii, ia, + d, m, + fd, td, + m1, mc, mi, mr, rm, + rmi, mri, mrc, + rm0, vm, vmi, rvm, rvmr, rvmi, mvr, rmv, + // zig fmt: on +}; + +pub const Op = enum { + // zig fmt: off + none, + unity, + imm8, imm16, imm32, imm64, + imm8s, imm16s, imm32s, + al, ax, eax, rax, + cl, dx, + rip, eip, ip, + r8, r16, r32, r64, + rm8, rm16, rm32, rm64, + r32_m8, r32_m16, r64_m16, + m8, m16, m32, m64, m80, m128, m256, + rel8, rel16, rel32, + m, moffs, mrip8, + sreg, + st0, st, mm, mm_m64, + xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128, + ymm, ymm_m256, + cr, dr, + // zig fmt: on + + pub fn fromOperand(operand: Instruction.Operand, target: *const std.Target) Op { + return switch (operand) { + .none => .none, + + .reg => |reg| switch (reg.class()) { + .general_purpose => switch (reg) { + .al => .al, + .ax => .ax, + .eax => .eax, + .rax => .rax, + .cl => .cl, + .dx => .dx, + else => switch (reg.size().bitSize(target)) { + 8 => .r8, + 16 => .r16, + 32 => .r32, + 64 => .r64, + else => unreachable, + }, + }, + .gphi => .r8, + .segment => .sreg, + .x87 => switch (reg) { + .st0 => .st0, + else => .st, + }, + .mmx => .mm, + .sse => switch (reg) { + .xmm0 => .xmm0, + else => switch (reg.size().bitSize(target)) { + 128 => .xmm, + 256 => .ymm, + else => unreachable, + }, + }, + .ip => switch (reg) { + .rip => .rip, + .eip => .eip, + .ip => .ip, + else => unreachable, + }, + .cr => .cr, + .dr => .dr, + }, + + .mem => |mem| switch (mem) { + .moffs => .moffs, + .sib => switch (mem.bitSize(target)) { + 0 => .m, + 8 => .m8, + 16 => .m16, + 32 => .m32, + 64 => .m64, + 80 => .m80, + 128 => .m128, + 256 => .m256, + else => unreachable, + }, + .rip => switch (mem.bitSize(target)) { + 0, 8 => .mrip8, + 16 => .m16, + 32 => .m32, + 64 => .m64, + 80 => .m80, + 128 => .m128, + 256 => .m256, + else => unreachable, + }, + }, + + .imm => |imm| switch (imm) { + .signed => |x| if (x == 1) + .unity + else if (math.cast(i8, x)) |_| + .imm8s + else if (math.cast(i16, x)) |_| + .imm16s + else + .imm32s, + .unsigned => |x| if (x == 1) + .unity + else if (math.cast(i8, x)) |_| + .imm8s + else if (math.cast(u8, x)) |_| + .imm8 + else if (math.cast(i16, x)) |_| + .imm16s + else if (math.cast(u16, x)) |_| + .imm16 + else if (math.cast(i32, x)) |_| + .imm32s + else if (math.cast(u32, x)) |_| + .imm32 + else + .imm64, + }, + + .bytes => unreachable, + }; + } + + pub fn toReg(op: Op) Register { + return switch (op) { + else => .none, + .al => .al, + .ax => .ax, + .eax => .eax, + .rax => .rax, + .cl => .cl, + .dx => .dx, + .rip => .rip, + .eip => .eip, + .ip => .ip, + .st0 => .st0, + .xmm0 => .xmm0, + }; + } + + pub fn immBitSize(op: Op) u64 { + return switch (op) { + .none, .m, .moffs, .mrip8, .sreg => unreachable, + .al, .cl, .dx, .rip, .eip, .ip, .r8, .rm8, .r32_m8 => unreachable, + .ax, .r16, .rm16 => unreachable, + .eax, .r32, .rm32, .r32_m16 => unreachable, + .rax, .r64, .rm64, .r64_m16 => unreachable, + .st0, .st, .mm, .mm_m64 => unreachable, + .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable, + .ymm, .ymm_m256 => unreachable, + .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable, + .cr, .dr => unreachable, + .unity => 1, + .imm8, .imm8s, .rel8 => 8, + .imm16, .imm16s, .rel16 => 16, + .imm32, .imm32s, .rel32 => 32, + .imm64 => 64, + }; + } + + pub fn regBitSize(op: Op) u64 { + return switch (op) { + .none, .m, .moffs, .mrip8, .sreg => unreachable, + .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable, + .rel8, .rel16, .rel32 => unreachable, + .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable, + .al, .cl, .r8, .rm8 => 8, + .ax, .dx, .ip, .r16, .rm16 => 16, + .eax, .eip, .r32, .rm32, .r32_m8, .r32_m16 => 32, + .rax, .rip, .r64, .rm64, .r64_m16, .mm, .mm_m64, .cr, .dr => 64, + .st0, .st => 80, + .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => 128, + .ymm, .ymm_m256 => 256, + }; + } + + pub fn memBitSize(op: Op) u64 { + return switch (op) { + .none, .m, .moffs, .sreg => unreachable, + .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable, + .rel8, .rel16, .rel32 => unreachable, + .al, .cl, .r8, .ax, .dx, .ip, .r16, .eax, .eip, .r32, .rax, .rip, .r64 => unreachable, + .st0, .st, .mm, .xmm0, .xmm, .ymm => unreachable, + .cr, .dr => unreachable, + .mrip8, .m8, .rm8, .r32_m8, .xmm_m8 => 8, + .m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16, + .m32, .rm32, .xmm_m32 => 32, + .m64, .rm64, .mm_m64, .xmm_m64 => 64, + .m80 => 80, + .m128, .xmm_m128 => 128, + .m256, .ymm_m256 => 256, + }; + } + + pub fn isSigned(op: Op) bool { + return switch (op) { + .unity, .imm8, .imm16, .imm32, .imm64 => false, + .imm8s, .imm16s, .imm32s => true, + .rel8, .rel16, .rel32 => true, + else => unreachable, + }; + } + + pub fn isUnsigned(op: Op) bool { + return !op.isSigned(); + } + + pub fn isRegister(op: Op) bool { + // zig fmt: off + return switch (op) { + .al, .ax, .eax, .rax, + .cl, .dx, + .ip, .eip, .rip, + .r8, .r16, .r32, .r64, + .rm8, .rm16, .rm32, .rm64, + .r32_m8, .r32_m16, .r64_m16, + .st0, .st, .mm, .mm_m64, + .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128, + .ymm, .ymm_m256, + .cr, .dr, + => true, + else => false, + }; + // zig fmt: on + } + + pub fn isImmediate(op: Op) bool { + // zig fmt: off + return switch (op) { + .imm8, .imm16, .imm32, .imm64, + .imm8s, .imm16s, .imm32s, + .rel8, .rel16, .rel32, + .unity, + => true, + else => false, + }; + // zig fmt: on + } + + pub fn isMemory(op: Op) bool { + // zig fmt: off + return switch (op) { + .rm8, .rm16, .rm32, .rm64, + .r32_m8, .r32_m16, .r64_m16, + .m8, .m16, .m32, .m64, .m80, .m128, .m256, + .m, .moffs, .mrip8, + .mm_m64, + .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128, + .ymm_m256, + => true, + else => false, + }; + // zig fmt: on + } + + pub fn isSegmentRegister(op: Op) bool { + return switch (op) { + .moffs, .sreg => true, + else => false, + }; + } + + pub fn class(op: Op) bits.Register.Class { + return switch (op) { + else => unreachable, + .al, .ax, .eax, .rax, .cl, .dx => .general_purpose, + .r8, .r16, .r32, .r64 => .general_purpose, + .rm8, .rm16, .rm32, .rm64 => .general_purpose, + .r32_m8, .r32_m16, .r64_m16 => .general_purpose, + .sreg => .segment, + .st0, .st => .x87, + .mm, .mm_m64 => .mmx, + .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => .sse, + .ymm, .ymm_m256 => .sse, + .rip, .eip, .ip => .ip, + .cr => .cr, + .dr => .dr, + }; + } + + /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding. + pub fn isSubset(op: Op, target: Op) bool { + switch (op) { + .none, .m, .moffs, .sreg => return op == target, + else => { + if (op.isRegister() and target.isRegister()) { + return switch (target.toReg()) { + .none => op.class() == target.class() and op.regBitSize() == target.regBitSize(), + else => op == target, + }; + } + if (op.isMemory() and target.isMemory()) { + switch (target) { + .m => return true, + .moffs, .mrip8 => return op == target, + else => return op.memBitSize() == target.memBitSize(), + } + } + if (op.isImmediate() and target.isImmediate()) { + switch (target) { + .imm64 => if (op.immBitSize() <= 64) return true, + .imm32s, .rel32 => if (op.immBitSize() < 32 or (op.immBitSize() == 32 and op.isSigned())) + return true, + .imm32 => if (op.immBitSize() <= 32) return true, + .imm16s, .rel16 => if (op.immBitSize() < 16 or (op.immBitSize() == 16 and op.isSigned())) + return true, + .imm16 => if (op.immBitSize() <= 16) return true, + .imm8s, .rel8 => if (op.immBitSize() < 8 or (op.immBitSize() == 8 and op.isSigned())) + return true, + .imm8 => if (op.immBitSize() <= 8) return true, + else => {}, + } + return op == target; + } + return false; + }, + } + } +}; + +pub const Mode = enum { + // zig fmt: off + none, + short, long, + rex, rex_short, + wait, + vex_128_w0, vex_128_w1, vex_128_wig, + vex_256_w0, vex_256_w1, vex_256_wig, + vex_lig_w0, vex_lig_w1, vex_lig_wig, + vex_lz_w0, vex_lz_w1, vex_lz_wig, + // zig fmt: on + + pub fn isShort(mode: Mode) bool { + return switch (mode) { + .short, .rex_short => true, + else => false, + }; + } + + pub fn isLong(mode: Mode) bool { + return switch (mode) { + .long, + .vex_128_w1, + .vex_256_w1, + .vex_lig_w1, + .vex_lz_w1, + => true, + else => false, + }; + } + + pub fn isRex(mode: Mode) bool { + return switch (mode) { + else => false, + .rex, .rex_short => true, + }; + } + + pub fn isVex(mode: Mode) bool { + return switch (mode) { + // zig fmt: off + else => false, + .vex_128_w0, .vex_128_w1, .vex_128_wig, + .vex_256_w0, .vex_256_w1, .vex_256_wig, + .vex_lig_w0, .vex_lig_w1, .vex_lig_wig, + .vex_lz_w0, .vex_lz_w1, .vex_lz_wig, + => true, + // zig fmt: on + }; + } + + pub fn isVecLong(mode: Mode) bool { + return switch (mode) { + // zig fmt: off + else => unreachable, + .vex_128_w0, .vex_128_w1, .vex_128_wig, + .vex_lig_w0, .vex_lig_w1, .vex_lig_wig, + .vex_lz_w0, .vex_lz_w1, .vex_lz_wig, + => false, + .vex_256_w0, .vex_256_w1, .vex_256_wig, + => true, + // zig fmt: on + }; + } +}; + +pub const Feature = enum { + none, + @"32bit", + @"64bit", + adx, + aes, + @"aes avx", + avx, + avx2, + bmi, + bmi2, + cldemote, + clflushopt, + clwb, + cmov, + @"cmov x87", + crc32, + enqcmd, + f16c, + fma, + fsgsbase, + fxsr, + gfni, + @"gfni avx", + hreset, + @"invpcid 32bit", + @"invpcid 64bit", + kl, + lzcnt, + mmx, + movbe, + pclmul, + @"pclmul avx", + pku, + popcnt, + prefetch, + @"prefetchi 64bit", + prefetchwt1, + prfchw, + rdrnd, + rdseed, + @"rdpid 32bit", + @"rdpid 64bit", + sahf, + serialize, + shstk, + smap, + sse, + sse2, + sse3, + @"sse3 x87", + sse4_1, + sse4_2, + ssse3, + sha, + uintr, + vaes, + vpclmulqdq, + waitpkg, + widekl, + x87, +}; + +fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Operand) usize { + var inst: Instruction = .{ + .prefix = prefix, + .encoding = encoding, + .ops = @splat(.none), + }; + @memcpy(inst.ops[0..ops.len], ops); + + // By using a buffer with maximum length of encoded instruction, we can use + // the `end` field of the Writer for the count. + var buf: [16]u8 = undefined; + var trash: std.Io.Writer.Discarding = .init(&buf); + inst.encode(&trash.writer, .{ + .allow_frame_locs = true, + .allow_symbols = true, + }) catch { + // Since the function signature for encode() does not mention under what + // conditions it can fail, I have changed `unreachable` to `@panic` here. + // This is a TODO item since it indicates this function + // (`estimateInstructionLength`) has the wrong function signature. + @panic("unexpected failure to encode"); + }; + return trash.writer.end; +} + +const mnemonic_to_encodings_map = init: { + @setEvalBranchQuota(5_900); + const ModrmExt = u3; + const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, ModrmExt, Mode, Feature }; + const encodings: []const Entry = @import("encodings.zon"); + + const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len; + var mnemonic_map: [mnemonic_count][]Data = @splat(&.{}); + for (encodings) |entry| mnemonic_map[@intFromEnum(entry[0])].len += 1; + var data_storage: [encodings.len]Data = undefined; + var storage_index: usize = 0; + for (&mnemonic_map) |*value| { + value.ptr = data_storage[storage_index..].ptr; + storage_index += value.len; + } + var mnemonic_index: [mnemonic_count]usize = @splat(0); + const ops_len = @typeInfo(@FieldType(Data, "ops")).array.len; + const opc_len = @typeInfo(@FieldType(Data, "opc")).array.len; + for (encodings) |entry| { + const index = &mnemonic_index[@intFromEnum(entry[0])]; + mnemonic_map[@intFromEnum(entry[0])][index.*] = .{ + .op_en = entry[1], + .ops = (entry[2] ++ .{.none} ** (ops_len - entry[2].len)).*, + .opc_len = entry[3].len, + .opc = (entry[3] ++ .{undefined} ** (opc_len - entry[3].len)).*, + .modrm_ext = entry[4], + .mode = entry[5], + .feature = entry[6], + }; + index.* += 1; + } + const final_storage = data_storage; + var final_map: [mnemonic_count][]const Data = @splat(&.{}); + storage_index = 0; + for (&final_map, mnemonic_map) |*final_value, value| { + final_value.* = final_storage[storage_index..][0..value.len]; + storage_index += value.len; + } + break :init final_map; +}; diff --git a/src/codegen/x86_64/Lower.zig b/src/codegen/x86_64/Lower.zig new file mode 100644 index 0000000000000000000000000000000000000000..ca1fb1e428b672731bc57477bfe553e0b8ea2450 --- /dev/null +++ b/src/codegen/x86_64/Lower.zig @@ -0,0 +1,689 @@ +//! This file contains the functionality for lowering x86_64 MIR to Instructions + +target: *const std.Target, +allocator: std.mem.Allocator, +mir: Mir, +cc: std.builtin.CallingConvention, +err_msg: ?*Zcu.ErrorMsg = null, +src_loc: Zcu.LazySrcLoc, +result_insts_len: ResultInstIndex = undefined, +result_insts: [max_result_insts]Instruction = undefined, +result_relocs_len: ResultRelocIndex = undefined, +result_relocs: [max_result_relocs]Reloc = undefined, + +const max_result_insts = @max( + 1, // non-pseudo instructions + 2, // cmovcc: cmovcc \ cmovcc + 3, // setcc: setcc \ setcc \ logicop + 2, // jcc: jcc \ jcc + pseudo_probe_align_insts, + pseudo_probe_adjust_unrolled_max_insts, + pseudo_probe_adjust_setup_insts, + pseudo_probe_adjust_loop_insts, + abi.zigcc.callee_preserved_regs.len * 2, // push_regs/pop_regs + abi.Win64.callee_preserved_regs.len * 2, // push_regs/pop_regs + abi.SysV.callee_preserved_regs.len * 2, // push_regs/pop_regs +); +const max_result_relocs = @max( + 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea + 2, // jcc: jcc \ jcc + 2, // test \ jcc \ probe \ sub \ jmp + 1, // probe \ sub \ jcc +); + +const ResultInstIndex = std.math.IntFittingRange(0, max_result_insts); +const ResultRelocIndex = std.math.IntFittingRange(0, max_result_relocs); +pub const InstOpIndex = std.math.IntFittingRange( + 0, + @typeInfo(@FieldType(Instruction, "ops")).array.len, +); + +pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp +pub const pseudo_probe_adjust_unrolled_max_insts = + pseudo_probe_adjust_setup_insts + pseudo_probe_adjust_loop_insts; +pub const pseudo_probe_adjust_setup_insts = 2; // mov \ sub +pub const pseudo_probe_adjust_loop_insts = 3; // probe \ sub \ jcc + +pub const Error = error{ + OutOfMemory, + LowerFail, + InvalidInstruction, + CannotEncode, + CodegenFail, +} || codegen.GenerateSymbolError; + +pub const Reloc = struct { + lowered_inst_index: ResultInstIndex, + op_index: InstOpIndex, + target: Target, + off: i32, + + const Target = union(enum) { + inst: Mir.Inst.Index, + table, + nav: InternPool.Nav.Index, + uav: InternPool.Key.Ptr.BaseAddr.Uav, + lazy_sym: link.File.LazySymbol, + extern_func: Mir.NullTerminatedString, + }; +}; + +const Options = struct { allow_frame_locs: bool }; + +/// The returned slice is overwritten by the next call to lowerMir. +pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { + insts: []Instruction, + relocs: []const Reloc, +} { + lower.result_insts = undefined; + lower.result_relocs = undefined; + errdefer lower.result_insts = undefined; + errdefer lower.result_relocs = undefined; + lower.result_insts_len = 0; + lower.result_relocs_len = 0; + defer lower.result_insts_len = undefined; + defer lower.result_relocs_len = undefined; + + const inst = lower.mir.instructions.get(index); + switch (inst.tag) { + else => try lower.generic(inst), + .pseudo => switch (inst.ops) { + .pseudo_cmov_z_and_np_rr => { + assert(inst.data.rr.fixes == ._); + try lower.encode(.none, .cmovnz, &.{ + .{ .reg = inst.data.rr.r2 }, + .{ .reg = inst.data.rr.r1 }, + }); + try lower.encode(.none, .cmovnp, &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }); + }, + .pseudo_cmov_nz_or_p_rr => { + assert(inst.data.rr.fixes == ._); + try lower.encode(.none, .cmovnz, &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }); + try lower.encode(.none, .cmovp, &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }); + }, + .pseudo_cmov_nz_or_p_rm => { + assert(inst.data.rx.fixes == ._); + try lower.encode(.none, .cmovnz, &.{ + .{ .reg = inst.data.rx.r1 }, + .{ .mem = lower.mem(1, inst.data.rx.payload) }, + }); + try lower.encode(.none, .cmovp, &.{ + .{ .reg = inst.data.rx.r1 }, + .{ .mem = lower.mem(1, inst.data.rx.payload) }, + }); + }, + .pseudo_set_z_and_np_r => { + assert(inst.data.rr.fixes == ._); + try lower.encode(.none, .setz, &.{ + .{ .reg = inst.data.rr.r1 }, + }); + try lower.encode(.none, .setnp, &.{ + .{ .reg = inst.data.rr.r2 }, + }); + try lower.encode(.none, .@"and", &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }); + }, + .pseudo_set_z_and_np_m => { + assert(inst.data.rx.fixes == ._); + try lower.encode(.none, .setz, &.{ + .{ .mem = lower.mem(0, inst.data.rx.payload) }, + }); + try lower.encode(.none, .setnp, &.{ + .{ .reg = inst.data.rx.r1 }, + }); + try lower.encode(.none, .@"and", &.{ + .{ .mem = lower.mem(0, inst.data.rx.payload) }, + .{ .reg = inst.data.rx.r1 }, + }); + }, + .pseudo_set_nz_or_p_r => { + assert(inst.data.rr.fixes == ._); + try lower.encode(.none, .setnz, &.{ + .{ .reg = inst.data.rr.r1 }, + }); + try lower.encode(.none, .setp, &.{ + .{ .reg = inst.data.rr.r2 }, + }); + try lower.encode(.none, .@"or", &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }); + }, + .pseudo_set_nz_or_p_m => { + assert(inst.data.rx.fixes == ._); + try lower.encode(.none, .setnz, &.{ + .{ .mem = lower.mem(0, inst.data.rx.payload) }, + }); + try lower.encode(.none, .setp, &.{ + .{ .reg = inst.data.rx.r1 }, + }); + try lower.encode(.none, .@"or", &.{ + .{ .mem = lower.mem(0, inst.data.rx.payload) }, + .{ .reg = inst.data.rx.r1 }, + }); + }, + .pseudo_j_z_and_np_inst => { + assert(inst.data.inst.fixes == ._); + try lower.encode(.none, .jnz, &.{ + .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) }, + }); + try lower.encode(.none, .jnp, &.{ + .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, + }); + }, + .pseudo_j_nz_or_p_inst => { + assert(inst.data.inst.fixes == ._); + try lower.encode(.none, .jnz, &.{ + .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, + }); + try lower.encode(.none, .jp, &.{ + .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, + }); + }, + + .pseudo_probe_align_ri_s => { + try lower.encode(.none, .@"test", &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = .s(@bitCast(inst.data.ri.i)) }, + }); + try lower.encode(.none, .jz, &.{ + .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) }, + }); + try lower.encode(.none, .lea, &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .mem = Memory.initSib(.qword, .{ + .base = .{ .reg = inst.data.ri.r1 }, + .disp = -page_size, + }) }, + }); + try lower.encode(.none, .@"test", &.{ + .{ .mem = Memory.initSib(.dword, .{ + .base = .{ .reg = inst.data.ri.r1 }, + }) }, + .{ .reg = inst.data.ri.r1.to32() }, + }); + try lower.encode(.none, .jmp, &.{ + .{ .imm = lower.reloc(0, .{ .inst = index }, 0) }, + }); + assert(lower.result_insts_len == pseudo_probe_align_insts); + }, + .pseudo_probe_adjust_unrolled_ri_s => { + var offset = page_size; + while (offset < @as(i32, @bitCast(inst.data.ri.i))) : (offset += page_size) { + try lower.encode(.none, .@"test", &.{ + .{ .mem = Memory.initSib(.dword, .{ + .base = .{ .reg = inst.data.ri.r1 }, + .disp = -offset, + }) }, + .{ .reg = inst.data.ri.r1.to32() }, + }); + } + try lower.encode(.none, .sub, &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = .s(@bitCast(inst.data.ri.i)) }, + }); + assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts); + }, + .pseudo_probe_adjust_setup_rri_s => { + try lower.encode(.none, .mov, &.{ + .{ .reg = inst.data.rri.r2.to32() }, + .{ .imm = .s(@bitCast(inst.data.rri.i)) }, + }); + try lower.encode(.none, .sub, &.{ + .{ .reg = inst.data.rri.r1 }, + .{ .reg = inst.data.rri.r2 }, + }); + assert(lower.result_insts_len == pseudo_probe_adjust_setup_insts); + }, + .pseudo_probe_adjust_loop_rr => { + try lower.encode(.none, .@"test", &.{ + .{ .mem = Memory.initSib(.dword, .{ + .base = .{ .reg = inst.data.rr.r1 }, + .scale_index = .{ .scale = 1, .index = inst.data.rr.r2 }, + .disp = -page_size, + }) }, + .{ .reg = inst.data.rr.r1.to32() }, + }); + try lower.encode(.none, .sub, &.{ + .{ .reg = inst.data.rr.r2 }, + .{ .imm = .s(page_size) }, + }); + try lower.encode(.none, .jae, &.{ + .{ .imm = lower.reloc(0, .{ .inst = index }, 0) }, + }); + assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts); + }, + .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst), + .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst), + + .pseudo_cfi_def_cfa_ri_s => try lower.encode(.directive, .@".cfi_def_cfa", &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, + }), + .pseudo_cfi_def_cfa_register_r => try lower.encode(.directive, .@".cfi_def_cfa_register", &.{ + .{ .reg = inst.data.r.r1 }, + }), + .pseudo_cfi_def_cfa_offset_i_s => try lower.encode(.directive, .@".cfi_def_cfa_offset", &.{ + .{ .imm = lower.imm(.i_s, inst.data.i.i) }, + }), + .pseudo_cfi_adjust_cfa_offset_i_s => try lower.encode(.directive, .@".cfi_adjust_cfa_offset", &.{ + .{ .imm = lower.imm(.i_s, inst.data.i.i) }, + }), + .pseudo_cfi_offset_ri_s => try lower.encode(.directive, .@".cfi_offset", &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, + }), + .pseudo_cfi_val_offset_ri_s => try lower.encode(.directive, .@".cfi_val_offset", &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, + }), + .pseudo_cfi_rel_offset_ri_s => try lower.encode(.directive, .@".cfi_rel_offset", &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = lower.imm(.ri_s, inst.data.ri.i) }, + }), + .pseudo_cfi_register_rr => try lower.encode(.directive, .@".cfi_register", &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }), + .pseudo_cfi_restore_r => try lower.encode(.directive, .@".cfi_restore", &.{ + .{ .reg = inst.data.r.r1 }, + }), + .pseudo_cfi_undefined_r => try lower.encode(.directive, .@".cfi_undefined", &.{ + .{ .reg = inst.data.r.r1 }, + }), + .pseudo_cfi_same_value_r => try lower.encode(.directive, .@".cfi_same_value", &.{ + .{ .reg = inst.data.r.r1 }, + }), + .pseudo_cfi_remember_state_none => try lower.encode(.directive, .@".cfi_remember_state", &.{}), + .pseudo_cfi_restore_state_none => try lower.encode(.directive, .@".cfi_restore_state", &.{}), + .pseudo_cfi_escape_bytes => try lower.encode(.directive, .@".cfi_escape", &.{ + .{ .bytes = inst.data.bytes.get(lower.mir) }, + }), + + .pseudo_dbg_prologue_end_none, + .pseudo_dbg_line_stmt_line_column, + .pseudo_dbg_line_line_column, + .pseudo_dbg_epilogue_begin_none, + .pseudo_dbg_enter_block_none, + .pseudo_dbg_leave_block_none, + .pseudo_dbg_enter_inline_func, + .pseudo_dbg_leave_inline_func, + .pseudo_dbg_arg_none, + .pseudo_dbg_arg_i_s, + .pseudo_dbg_arg_i_u, + .pseudo_dbg_arg_i_64, + .pseudo_dbg_arg_ro, + .pseudo_dbg_arg_fa, + .pseudo_dbg_arg_m, + .pseudo_dbg_arg_val, + .pseudo_dbg_var_args_none, + .pseudo_dbg_var_none, + .pseudo_dbg_var_i_s, + .pseudo_dbg_var_i_u, + .pseudo_dbg_var_i_64, + .pseudo_dbg_var_ro, + .pseudo_dbg_var_fa, + .pseudo_dbg_var_m, + .pseudo_dbg_var_val, + + .pseudo_dead_none, + => {}, + else => unreachable, + }, + } + + return .{ + .insts = lower.result_insts[0..lower.result_insts_len], + .relocs = lower.result_relocs[0..lower.result_relocs_len], + }; +} + +pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { + @branchHint(.cold); + assert(lower.err_msg == null); + lower.err_msg = try .create(lower.allocator, lower.src_loc, format, args); + return error.LowerFail; +} + +pub fn imm(lower: *const Lower, ops: Mir.Inst.Ops, i: u32) Immediate { + return switch (ops) { + .rri_s, + .ri_s, + .i_s, + .mi_s, + .rmi_s, + .pseudo_dbg_arg_i_s, + .pseudo_dbg_var_i_s, + => .s(@bitCast(i)), + + .ii, + .ir, + .rrri, + .rri_u, + .ri_u, + .i_u, + .mi_u, + .rmi, + .rmi_u, + .mri, + .rrm, + .rrmi, + .pseudo_dbg_arg_i_u, + .pseudo_dbg_var_i_u, + => .u(i), + + .ri_64, + => .u(lower.mir.extraData(Mir.Imm64, i).data.decode()), + + .pseudo_dbg_arg_i_64, + .pseudo_dbg_var_i_64, + => unreachable, + + else => unreachable, + }; +} + +fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory { + var m = lower.mir.resolveMemoryExtra(payload).decode(); + switch (m) { + .sib => |*sib| switch (sib.base) { + .none, .reg, .frame => {}, + .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed, + .rip_inst => |inst_index| sib.disp = lower.reloc(op_index, .{ .inst = inst_index }, sib.disp).signed, + .nav => |nav| sib.disp = lower.reloc(op_index, .{ .nav = nav }, sib.disp).signed, + .uav => |uav| sib.disp = lower.reloc(op_index, .{ .uav = uav }, sib.disp).signed, + .lazy_sym => |lazy_sym| sib.disp = lower.reloc(op_index, .{ .lazy_sym = lazy_sym }, sib.disp).signed, + .extern_func => |extern_func| sib.disp = lower.reloc(op_index, .{ .extern_func = extern_func }, sib.disp).signed, + }, + else => {}, + } + return m; +} + +fn reloc(lower: *Lower, op_index: InstOpIndex, target: Reloc.Target, off: i32) Immediate { + lower.result_relocs[lower.result_relocs_len] = .{ + .lowered_inst_index = lower.result_insts_len, + .op_index = op_index, + .target = target, + .off = off, + }; + lower.result_relocs_len += 1; + return .s(0); +} + +fn encode(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void { + lower.result_insts[lower.result_insts_len] = try .new(prefix, mnemonic, ops, lower.target); + lower.result_insts_len += 1; +} + +const inst_tags_len = @typeInfo(Mir.Inst.Tag).@"enum".fields.len; +const inst_fixes_len = @typeInfo(Mir.Inst.Fixes).@"enum".fields.len; +/// Lookup table, indexed by `@intFromEnum(inst.tag) * inst_fixes_len + @intFromEnum(fixes)`. +/// The value is the resulting `Mnemonic`, or `null` if the combination is not valid. +const mnemonic_table: [inst_tags_len * inst_fixes_len]?Mnemonic = table: { + @setEvalBranchQuota(80_000); + var table: [inst_tags_len * inst_fixes_len]?Mnemonic = undefined; + for (0..inst_fixes_len) |fixes_i| { + const fixes: Mir.Inst.Fixes = @enumFromInt(fixes_i); + const prefix, const suffix = affix: { + const pattern = if (std.mem.indexOfScalar(u8, @tagName(fixes), ' ')) |i| + @tagName(fixes)[i + 1 ..] + else + @tagName(fixes); + const wildcard_idx = std.mem.indexOfScalar(u8, pattern, '_').?; + break :affix .{ pattern[0..wildcard_idx], pattern[wildcard_idx + 1 ..] }; + }; + for (0..inst_tags_len) |inst_tag_i| { + const inst_tag: Mir.Inst.Tag = @enumFromInt(inst_tag_i); + const name = prefix ++ @tagName(inst_tag) ++ suffix; + const idx = inst_tag_i * inst_fixes_len + fixes_i; + table[idx] = if (@hasField(Mnemonic, name)) @field(Mnemonic, name) else null; + } + } + break :table table; +}; + +fn generic(lower: *Lower, inst: Mir.Inst) Error!void { + @setEvalBranchQuota(2_000); + const fixes = switch (inst.ops) { + .none => inst.data.none.fixes, + .inst => inst.data.inst.fixes, + .i_s, .i_u => inst.data.i.fixes, + .ii => inst.data.ii.fixes, + .r => inst.data.r.fixes, + .rr => inst.data.rr.fixes, + .rrr => inst.data.rrr.fixes, + .rrrr => inst.data.rrrr.fixes, + .rrri => inst.data.rrri.fixes, + .rri_s, .rri_u => inst.data.rri.fixes, + .ri_s, .ri_u, .ri_64, .ir => inst.data.ri.fixes, + .rm, .rmi_s, .rmi_u, .mr => inst.data.rx.fixes, + .mrr, .rrm, .rmr => inst.data.rrx.fixes, + .rmi, .mri => inst.data.rix.fixes, + .rrmr => inst.data.rrrx.fixes, + .rrmi => inst.data.rrix.fixes, + .mi_u, .mi_s => inst.data.x.fixes, + .m => inst.data.x.fixes, + .nav, .uav, .lazy_sym, .extern_func => ._, + else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}), + }; + try lower.encode(switch (fixes) { + inline else => |tag| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) |space| + @field(Prefix, @tagName(tag)[0..space]) + else + .none, + }, mnemonic: { + if (mnemonic_table[@intFromEnum(inst.tag) * inst_fixes_len + @intFromEnum(fixes)]) |mnemonic| { + break :mnemonic mnemonic; + } + // This combination is invalid; make the theoretical mnemonic name and emit an error with it. + const fixes_name = @tagName(fixes); + const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) |i| i + " ".len else 0..]; + const wildcard_index = std.mem.indexOfScalar(u8, pattern, '_').?; + return lower.fail("unsupported mnemonic: '{s}{s}{s}'", .{ + pattern[0..wildcard_index], + @tagName(inst.tag), + pattern[wildcard_index + "_".len ..], + }); + }, switch (inst.ops) { + .none => &.{}, + .inst => &.{ + .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, + }, + .i_s, .i_u => &.{ + .{ .imm = lower.imm(inst.ops, inst.data.i.i) }, + }, + .ii => &.{ + .{ .imm = lower.imm(inst.ops, inst.data.ii.i1) }, + .{ .imm = lower.imm(inst.ops, inst.data.ii.i2) }, + }, + .ir => &.{ + .{ .imm = lower.imm(inst.ops, inst.data.ri.i) }, + .{ .reg = inst.data.ri.r1 }, + }, + .r => &.{ + .{ .reg = inst.data.r.r1 }, + }, + .rr => &.{ + .{ .reg = inst.data.rr.r1 }, + .{ .reg = inst.data.rr.r2 }, + }, + .rrr => &.{ + .{ .reg = inst.data.rrr.r1 }, + .{ .reg = inst.data.rrr.r2 }, + .{ .reg = inst.data.rrr.r3 }, + }, + .rrrr => &.{ + .{ .reg = inst.data.rrrr.r1 }, + .{ .reg = inst.data.rrrr.r2 }, + .{ .reg = inst.data.rrrr.r3 }, + .{ .reg = inst.data.rrrr.r4 }, + }, + .rrri => &.{ + .{ .reg = inst.data.rrri.r1 }, + .{ .reg = inst.data.rrri.r2 }, + .{ .reg = inst.data.rrri.r3 }, + .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) }, + }, + .ri_s, .ri_u, .ri_64 => &.{ + .{ .reg = inst.data.ri.r1 }, + .{ .imm = lower.imm(inst.ops, inst.data.ri.i) }, + }, + .rri_s, .rri_u => &.{ + .{ .reg = inst.data.rri.r1 }, + .{ .reg = inst.data.rri.r2 }, + .{ .imm = lower.imm(inst.ops, inst.data.rri.i) }, + }, + .m => &.{ + .{ .mem = lower.mem(0, inst.data.x.payload) }, + }, + .mi_s, .mi_u => &.{ + .{ .mem = lower.mem(0, inst.data.x.payload + 1) }, + .{ .imm = lower.imm( + inst.ops, + lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm, + ) }, + }, + .rm => &.{ + .{ .reg = inst.data.rx.r1 }, + .{ .mem = lower.mem(1, inst.data.rx.payload) }, + }, + .rmr => &.{ + .{ .reg = inst.data.rrx.r1 }, + .{ .mem = lower.mem(1, inst.data.rrx.payload) }, + .{ .reg = inst.data.rrx.r2 }, + }, + .rmi => &.{ + .{ .reg = inst.data.rix.r1 }, + .{ .mem = lower.mem(1, inst.data.rix.payload) }, + .{ .imm = lower.imm(inst.ops, inst.data.rix.i) }, + }, + .rmi_s, .rmi_u => &.{ + .{ .reg = inst.data.rx.r1 }, + .{ .mem = lower.mem(1, inst.data.rx.payload + 1) }, + .{ .imm = lower.imm( + inst.ops, + lower.mir.extraData(Mir.Imm32, inst.data.rx.payload).data.imm, + ) }, + }, + .mr => &.{ + .{ .mem = lower.mem(0, inst.data.rx.payload) }, + .{ .reg = inst.data.rx.r1 }, + }, + .mrr => &.{ + .{ .mem = lower.mem(0, inst.data.rrx.payload) }, + .{ .reg = inst.data.rrx.r1 }, + .{ .reg = inst.data.rrx.r2 }, + }, + .mri => &.{ + .{ .mem = lower.mem(0, inst.data.rix.payload) }, + .{ .reg = inst.data.rix.r1 }, + .{ .imm = lower.imm(inst.ops, inst.data.rix.i) }, + }, + .rrm => &.{ + .{ .reg = inst.data.rrx.r1 }, + .{ .reg = inst.data.rrx.r2 }, + .{ .mem = lower.mem(2, inst.data.rrx.payload) }, + }, + .rrmr => &.{ + .{ .reg = inst.data.rrrx.r1 }, + .{ .reg = inst.data.rrrx.r2 }, + .{ .mem = lower.mem(2, inst.data.rrrx.payload) }, + .{ .reg = inst.data.rrrx.r3 }, + }, + .rrmi => &.{ + .{ .reg = inst.data.rrix.r1 }, + .{ .reg = inst.data.rrix.r2 }, + .{ .mem = lower.mem(2, inst.data.rrix.payload) }, + .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) }, + }, + .nav => &.{ + .{ .imm = lower.reloc(0, .{ .nav = inst.data.nav.index }, inst.data.nav.off) }, + }, + .uav => &.{ + .{ .imm = lower.reloc(0, .{ .uav = inst.data.uav }, 0) }, + }, + .lazy_sym => &.{ + .{ .imm = lower.reloc(0, .{ .lazy_sym = inst.data.lazy_sym }, 0) }, + }, + .extern_func => &.{ + .{ .imm = lower.reloc(0, .{ .extern_func = inst.data.extern_func }, 0) }, + }, + else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }), + }); +} + +fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void { + const callee_preserved_regs = abi.getCalleePreservedRegs(lower.cc); + var off: i32 = switch (mnemonic) { + .push => 0, + .pop => undefined, + else => unreachable, + }; + { + var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) { + .push => .reverse, + .pop => .forward, + else => unreachable, + } }); + while (it.next()) |i| { + try lower.encode(.none, mnemonic, &.{.{ + .reg = callee_preserved_regs[i], + }}); + switch (mnemonic) { + .push => off -= 8, + .pop => {}, + else => unreachable, + } + } + } + switch (mnemonic) { + .push => { + var it = inst.data.reg_list.iterator(.{}); + while (it.next()) |i| { + try lower.encode(.directive, .@".cfi_rel_offset", &.{ + .{ .reg = callee_preserved_regs[i] }, + .{ .imm = .s(off) }, + }); + off += 8; + } + assert(off == 0); + }, + .pop => {}, + else => unreachable, + } +} + +const page_size: i32 = 1 << 12; + +const abi = @import("abi.zig"); +const assert = std.debug.assert; +const bits = @import("bits.zig"); +const codegen = @import("../../codegen.zig"); +const encoder = @import("encoder.zig"); +const link = @import("../../link.zig"); +const std = @import("std"); + +const Immediate = Instruction.Immediate; +const Instruction = encoder.Instruction; +const InternPool = @import("../../InternPool.zig"); +const Lower = @This(); +const Memory = Instruction.Memory; +const Mir = @import("Mir.zig"); +const Mnemonic = Instruction.Mnemonic; +const Zcu = @import("../../Zcu.zig"); +const Operand = Instruction.Operand; +const Prefix = Instruction.Prefix; +const Register = bits.Register; +const Type = @import("../../Type.zig"); diff --git a/src/codegen/x86_64/Mir.zig b/src/codegen/x86_64/Mir.zig new file mode 100644 index 0000000000000000000000000000000000000000..caf41ffb392d07b8a455f09181c77ffb8a679d7e --- /dev/null +++ b/src/codegen/x86_64/Mir.zig @@ -0,0 +1,2131 @@ +//! Machine Intermediate Representation. +//! This data is produced by x86_64 Codegen and consumed by x86_64 Isel. +//! These instructions have a 1:1 correspondence with machine code instructions +//! for the target. MIR can be lowered to source-annotated textual assembly code +//! instructions, or it can be lowered to machine code. +//! The main purpose of MIR is to postpone the assignment of offsets until Isel, +//! so that, for example, the smaller encodings of jump instructions can be used. + +instructions: std.MultiArrayList(Inst).Slice, +/// The meaning of this data is determined by `Inst.Tag` value. +extra: []const u32, +string_bytes: []const u8, +locals: []const Local, +table: []const Inst.Index, +/// Optional data which, when present, can be used to accelerate encoding speed. +memoized_encodings: []const u0 = &.{}, +frame_locs: std.MultiArrayList(FrameLoc).Slice, + +pub const Inst = struct { + tag: Tag, + ops: Ops, + data: Data, + + pub const Index = u32; + + pub const Fixes = enum(u8) { + /// ___ + @"_", + + /// ___ 0 + _0, + /// ___ 1 + _1, + /// ___ 2 + _2, + /// ___ 3 + _3, + /// ___ 4 + _4, + + /// ___ Demote + _demote, + /// ___ Flush + _flush, + /// ___ Flush Optimized + _flushopt, + /// ___ Instructions With T0 Hint + _it0, + /// ___ Instructions With T0 Hint + _it1, + /// ___ With NTA Hint + _nta, + /// System Call ___ + sys_, + /// ___ With T0 Hint + _t0, + /// ___ With T1 Hint + _t1, + /// ___ With T2 Hint + _t2, + /// ___ Write Back + _wb, + /// ___ With Intent to Write and T1 Hint + _wt1, + + /// ___ crement Shadow Stack Pointer Doubleword + _csspd, + /// ___ crement Shadow Stack Pointer Quadword + _csspq, + /// ___ FS Segment Base + _fsbase, + /// ___ GS + _gs, + /// ___ GS Segment Base + _gsbase, + /// ___ Model Specific Register + _msr, + /// ___ MXCSR + _mxcsr, + /// ___ Processor ID + _pid, + /// ___ Protection Key Rights For User Pages + _pkru, + /// ___ Performance-Monitoring Counters + _pmc, + /// ___ Random Number + _rand, + /// ___ r Busy Flag in a Supervisor Shadow Stack token + _rssbsy, + /// ___ Random Seed + _seed, + /// ___ Shadow Stack Doubleword + _ssd, + /// ___ Shadow Stack Quadword + _ssq, + /// ___ Shadow Stack Pointer Doubleword + _sspd, + /// ___ Shadow Stack Pointer Quadword + _sspq, + /// ___ Time-Stamp Counter + _tsc, + /// ___ Time-Stamp Counter And Processor ID + _tscp, + /// ___ User Shadow Stack Doubleword + _ussd, + /// ___ User Shadow Stack Quadword + _ussq, + /// VEX-Encoded ___ MXCSR + v_mxcsr, + + /// Byte ___ + b_, + /// Interrupt ___ + /// Integer ___ + i_, + /// Interrupt ___ Word + i_w, + /// Interrupt ___ Doubleword + i_d, + /// Interrupt ___ Quadword + i_q, + /// User-Interrupt ___ + ui_, + + /// ___ mp + _mp, + /// ___ if CX register is 0 + _cxz, + /// ___ if ECX register is 0 + _ecxz, + /// ___ if RCX register is 0 + _rcxz, + + /// ___ Addition + _a, + /// ___ Subtraction + _s, + /// ___ Multiply + _m, + /// ___ Division + _d, + + /// ___ Without Affecting Flags + _x, + /// ___ Left + _l, + /// ___ Left Double + _ld, + /// ___ Left Without Affecting Flags + _lx, + /// ___ Mask + _msk, + /// ___ Right + /// ___ For Reading + /// ___ Register + _r, + /// ___ Right Double + _rd, + /// ___ Right Without Affecting Flags + _rx, + + /// ___ Forward + _f, + /// ___ Reverse + //_r, + + /// ___ Above + //_a, + /// ___ Above Or Equal + _ae, + /// ___ Below + _b, + /// ___ Below Or Equal + /// ___ Big Endian + _be, + /// ___ Carry + /// ___ Carry Flag + _c, + /// ___ Equal + _e, + /// ___ Greater + _g, + /// ___ Greater Or Equal + _ge, + /// ___ Less + //_l, + /// ___ Less Or Equal + _le, + /// ___ Not Above + _na, + /// ___ Not Above Or Equal + _nae, + /// ___ Not Below + _nb, + /// ___ Not Below Or Equal + _nbe, + /// ___ Not Carry + _nc, + /// ___ Not Equal + _ne, + /// ___ Not Greater + _ng, + /// ___ Not Greater Or Equal + _nge, + /// ___ Not Less + _nl, + /// ___ Not Less Or Equal + _nle, + /// ___ Not Overflow + _no, + /// ___ Not Parity + _np, + /// ___ Not Sign + _ns, + /// ___ Not Zero + _nz, + /// ___ Overflow + _o, + /// ___ Parity + _p, + /// ___ Parity Even + _pe, + /// ___ Parity Odd + _po, + /// ___ Sign + //_s, + /// ___ Zero + _z, + /// ___ Alignment Check Flag + _ac, + /// ___ Direction Flag + //_d, + /// ___ Interrupt Flag + _i, + /// ___ Task-Switched Flag In CR0 + _ts, + /// ___ User Interrupt Flag + _ui, + + /// ___ Byte + //_b, + /// ___ Word + /// ___ For Writing + /// ___ With Intent to Write + _w, + /// ___ Doubleword + //_d, + /// ___ Double Quadword to Quadword + _dq2q, + /// ___ QuadWord + _q, + /// ___ Quadword to Double Quadword + _q2dq, + + /// ___ String + //_s, + /// ___ String Byte + _sb, + /// ___ String Word + _sw, + /// ___ String Doubleword + _sd, + /// ___ String Quadword + _sq, + + /// Repeat ___ String + @"rep _s", + /// Repeat ___ String Byte + @"rep _sb", + /// Repeat ___ String Word + @"rep _sw", + /// Repeat ___ String Doubleword + @"rep _sd", + /// Repeat ___ String Quadword + @"rep _sq", + + /// Repeat Equal ___ String + @"repe _s", + /// Repeat Equal ___ String Byte + @"repe _sb", + /// Repeat Equal ___ String Word + @"repe _sw", + /// Repeat Equal ___ String Doubleword + @"repe _sd", + /// Repeat Equal ___ String Quadword + @"repe _sq", + + /// Repeat Not Equal ___ String + @"repne _s", + /// Repeat Not Equal ___ String Byte + @"repne _sb", + /// Repeat Not Equal ___ String Word + @"repne _sw", + /// Repeat Not Equal ___ String Doubleword + @"repne _sd", + /// Repeat Not Equal ___ String Quadword + @"repne _sq", + + /// Repeat Not Zero ___ String + @"repnz _s", + /// Repeat Not Zero ___ String Byte + @"repnz _sb", + /// Repeat Not Zero ___ String Word + @"repnz _sw", + /// Repeat Not Zero ___ String Doubleword + @"repnz _sd", + /// Repeat Not Zero ___ String Quadword + @"repnz _sq", + + /// Repeat Zero ___ String + @"repz _s", + /// Repeat Zero ___ String Byte + @"repz _sb", + /// Repeat Zero ___ String Word + @"repz _sw", + /// Repeat Zero ___ String Doubleword + @"repz _sd", + /// Repeat Zero ___ String Quadword + @"repz _sq", + + /// Locked ___ + @"lock _", + /// ___ And Complement + //_c, + /// Locked ___ And Complement + @"lock _c", + /// ___ And Reset + //_r, + /// Locked ___ And Reset + @"lock _r", + /// ___ And Set + //_s, + /// Locked ___ And Set + @"lock _s", + /// ___ 8 Bytes + _8b, + /// Locked ___ 8 Bytes + @"lock _8b", + /// ___ 16 Bytes + _16b, + /// Locked ___ 16 Bytes + @"lock _16b", + + /// Float ___ + f_, + /// Float ___ +1.0 + /// Float ___ 1 + f_1, + /// Float ___ Below + f_b, + /// Float ___ Below Or Equal + f_be, + /// Float ___ Control Word + f_cw, + /// Float ___ Equal + f_e, + /// Float ___ Environment + f_env, + /// Float ___ log_2(e) + f_l2e, + /// Float ___ log_2(10) + f_l2t, + /// Float ___ log_10(2) + f_lg2, + /// Float ___ log_e(2) + f_ln2, + /// Float ___ Not Below + f_nb, + /// Float ___ Not Below Or Equal + f_nbe, + /// Float ___ Not Equal + f_ne, + /// Float ___ Not Unordered + f_nu, + /// Float ___ Pop + f_p, + /// Float ___ +1 + f_p1, + /// Float ___ Ï€ + f_pi, + /// Float ___ Pop Pop + f_pp, + /// Float ___ crement Stack-Top Pointer + f_cstp, + /// Float ___ Status Word + f_sw, + /// Float ___ Unordered + f_u, + /// Float ___ +0.0 + f_z, + /// Float BCD ___ + fb_, + /// Float BCD ___ Pop + fb_p, + /// Float And Integer ___ + fi_, + /// Float And Integer ___ Pop + fi_p, + /// Float No Wait ___ + fn_, + /// Float No Wait ___ Control Word + fn_cw, + /// Float No Wait ___ Environment + fn_env, + /// Float No Wait ___ status word + fn_sw, + /// Float Extended ___ + fx_, + /// Float Extended ___ 64 + fx_64, + + /// ___ in 32-bit and Compatibility Mode + _32, + /// ___ in 64-bit Mode + _64, + + /// Packed ___ + p_, + /// Packed ___ Byte + p_b, + /// Packed ___ Word + p_w, + /// Packed ___ Doubleword + p_d, + /// Packed ___ Quadword + p_q, + /// Packed ___ Double Quadword + /// Packed ___ Doubleword to Quadword + p_dq, + /// Packed ___ Unsigned Doubleword to Quadword + p_udq, + /// Packed Carry-Less ___ Quadword to Double Quadword + pcl_qdq, + /// Packed Half ___ Doubleword + ph_d, + /// Packed Half ___ Saturate Word + ph_sw, + /// Packed Half ___ Word + ph_w, + /// ___ Aligned Packed Integer Values + _dqa, + /// ___ Unaligned Packed Integer Values + _dqu, + + /// ___ Scalar Single-Precision Values + _ss, + /// ___ Packed Single-Precision Values + _ps, + /// ___ Scalar Double-Precision Values + //_sd, + /// ___ Packed Double-Precision Values + _pd, + /// Half ___ Packed Single-Precision Values + h_ps, + /// Half ___ Packed Double-Precision Values + h_pd, + + /// ___ Internal Caches + //_d, + /// ___ TLB Entries + _lpg, + /// ___ Process-Context Identifier + _pcid, + + /// Load ___ + l_, + /// Memory ___ + m_, + /// Store ___ + s_, + /// Timed ___ + t_, + /// User Level Monitor ___ + um_, + + /// VEX-Encoded ___ + v_, + /// VEX-Encoded ___ Byte + v_b, + /// VEX-Encoded ___ Word + v_w, + /// VEX-Encoded ___ Doubleword + v_d, + /// VEX-Encoded ___ Quadword + v_q, + /// VEX-Encoded ___ Aligned Packed Integer Values + v_dqa, + /// VEX-Encoded ___ Unaligned Packed Integer Values + v_dqu, + /// VEX-Encoded ___ Integer Data + v_i128, + /// VEX-Encoded Packed ___ + vp_, + /// VEX-Encoded Packed ___ Byte + vp_b, + /// VEX-Encoded Packed ___ Word + vp_w, + /// VEX-Encoded Packed ___ Doubleword + vp_d, + /// VEX-Encoded Packed ___ Quadword + vp_q, + /// VEX-Encoded Packed ___ Double Quadword + /// VEX-Encoded Packed ___ Doubleword to Quadword + vp_dq, + /// VEX-Encoded Packed ___ Unsigned Doubleword to Quadword + vp_udq, + /// VEx-Encoded Packed Carry-Less ___ Quadword to Double Quadword + vpcl_qdq, + /// VEX-Encoded Packed Half ___ Doubleword + vph_d, + /// VEX-Encoded Packed Half ___ Saturate Word + vph_sw, + /// VEX-Encoded Packed Half ___ Word + vph_w, + /// VEX-Encoded ___ Scalar Single-Precision Values + v_ss, + /// VEX-Encoded ___ Packed Single-Precision Values + v_ps, + /// VEX-Encoded ___ Scalar Double-Precision Values + v_sd, + /// VEX-Encoded ___ Packed Double-Precision Values + v_pd, + /// VEX-Encoded ___ 128-Bits Of Floating-Point Data + v_f128, + /// VEX-Encoded Half ___ Packed Single-Precision Values + vh_ps, + /// VEX-Encoded Half ___ Packed Double-Precision Values + vh_pd, + + /// ___ 128-bit key with key locker + _128, + /// ___ 256-bit key with key locker + _256, + /// ___ with key locker using 128-bit key + _128kl, + /// ___ with key locker using 256-bit key + _256kl, + /// ___ with key locker on 8 blocks using 128-bit key + _wide128kl, + /// ___ with key locker on 8 blocks using 256-bit key + _wide256kl, + + /// Mask ___ Byte + k_b, + /// Mask ___ Word + k_w, + /// Mask ___ Doubleword + k_d, + /// Mask ___ Quadword + k_q, + + pub fn fromCond(cc: bits.Condition) Fixes { + return switch (cc) { + inline else => |cc_tag| @field(Fixes, "_" ++ @tagName(cc_tag)), + .z_and_np, .nz_or_p => unreachable, + }; + } + }; + + pub const Tag = enum(u8) { + // General-purpose + /// ASCII adjust al after addition + /// ASCII adjust ax before division + /// ASCII adjust ax after multiply + /// ASCII adjust al after subtraction + aa, + /// Add with carry + /// Unsigned integer addition of two operands with carry flag + adc, + /// Add + /// Add packed integers + /// Add packed single-precision floating-point values + /// Add scalar single-precision floating-point values + /// Add packed double-precision floating-point values + /// Add scalar double-precision floating-point values + /// Packed single-precision floating-point horizontal add + /// Packed double-precision floating-point horizontal add + /// Packed horizontal add + /// Packed horizontal add and saturate + add, + /// Logical and + /// Bitwise logical and of packed single-precision floating-point values + /// Bitwise logical and of packed double-precision floating-point values + @"and", + /// Adjust RPL field of segment selector + arpl, + /// Bit scan forward + /// Bit scan reverse + bs, + /// Byte swap + /// Swap GS base register + swap, + /// Bit test + /// Bit test and complement + /// Bit test and reset + /// Bit test and set + bt, + /// Check array index against bounds + bound, + /// Call + /// Fast system call + call, + /// Convert byte to word + cbw, + /// Convert doubleword to quadword + cdq, + /// Convert doubleword to quadword + cdqe, + /// Clear AC flag in EFLAGS register + /// Clear carry flag + /// Clear direction flag + /// Clear interrupt flag + /// Clear task-switched flag in CR0 + /// Clear user interrupt flag + /// Cache line demote + /// Flush cache line + /// Flush cache line optimized + /// Clear busy flag in a supervisor shadow stack token + /// Cache line write back + cl, + /// Complement carry flag + cmc, + /// Conditional move + cmov, + /// Logical compare + /// Compare string + /// Compare scalar single-precision floating-point values + /// Compare scalar double-precision floating-point values + cmp, + /// Compare and exchange + /// Compare and exchange bytes + cmpxchg, + /// CPU identification + cpuid, + /// Convert doubleword to quadword + cqo, + /// Convert word to doubleword + cwd, + /// Convert word to doubleword + cwde, + /// Decimal adjust AL after addition + /// Decimal adjust AL after subtraction + da, + /// Decrement by 1 + /// Decrement stack-top pointer + /// Decrement shadow stack pointer + de, + /// Unsigned division + /// Signed division + /// Divide + /// Divide packed single-precision floating-point values + /// Divide scalar single-precision floating-point values + /// Divide packed double-precision floating-point values + /// Divide scalar double-precision floating-point values + div, + /// Terminate and indirect branch in 32-bit and compatibility mode + /// Terminate and indirect branch in 64-bit mode + endbr, + /// Enqueue command + /// Enqueue command supervisor + enqcmd, + /// Make stack frame for procedure parameters + /// Fast system call + enter, + /// Fast return from fast system call + exit, + /// Load fence + /// Memory fence + /// Store fence + fence, + /// Halt + hlt, + /// History reset + hreset, + /// Input from port + /// Input from port to string + /// Increment by 1 + /// Increment stack-top pointer + /// Increment shadow stack pointer + in, + /// Call to interrupt procedure + int, + /// Invalidate internal caches + /// Invalidate TLB entries + /// Invalidate process-context identifier + inv, + /// Conditional jump + /// Jump + j, + /// Load status flags into AH register + lahf, + /// Load access right byte + lar, + /// Load effective address + lea, + /// High level procedure exit + leave, + /// Load global descriptor table register + lgdt, + /// Load interrupt descriptor table register + lidt, + /// Load local descriptor table register + lldt, + /// Load machine status word + lmsw, + /// Load string + lod, + /// Loop according to ECX counter + loop, + /// Load segment limit + lsl, + /// Load task register + ltr, + /// Count the number of leading zero bits + lzcnt, + /// Move + /// Move data from string to string + /// Move data after swapping bytes + /// Move scalar single-precision floating-point value + /// Move scalar double-precision floating-point value + /// Move doubleword + /// Move quadword + /// Move aligned packed integer values + /// Move unaligned packed integer values + /// Move quadword from XMM to MMX technology register + /// Move quadword from MMX technology to XMM register + mov, + /// Move with sign extension + movsx, + /// Move with zero extension + movzx, + /// Multiply + /// Signed multiplication + /// Multiply packed single-precision floating-point values + /// Multiply scalar single-precision floating-point values + /// Multiply packed double-precision floating-point values + /// Multiply scalar double-precision floating-point values + /// Multiply packed unsigned doubleword integers + /// Multiply packed doubleword integers + /// Carry-less multiplication quadword + mul, + /// Two's complement negation + neg, + /// No-op + /// No operation + nop, + /// One's complement negation + not, + /// Logical or + /// Bitwise logical or of packed single-precision floating-point values + /// Bitwise logical or of packed double-precision floating-point values + @"or", + /// Output to port + /// Output string to port + out, + /// Spin loop hint + /// Timed pause + pause, + /// Pop + pop, + /// Return the count of number of bits set to 1 + popcnt, + /// Pop stack into EFLAGS register + popf, + /// Push + push, + /// Push EFLAGS register onto the stack + pushf, + /// Rotate left through carry + /// Rotate right through carry + rc, + /// Read FS segment base + /// Read GS segment base + /// Read from model specific register + /// Read processor ID + /// Read protection key rights for user pages + /// Read performance-monitoring counters + /// Read random number + /// Read random seed + /// Read shadow stack pointer + /// Read time-stamp counter + /// Read time-stamp counter and processor ID + rd, + /// Return + /// Return from fast system call + /// Interrupt return + /// User-interrupt return + ret, + /// Rotate left + /// Rotate right + /// Rotate right logical without affecting flags + ro, + /// Resume from system management mode + rsm, + /// Arithmetic shift left + /// Arithmetic shift right + /// Shift left arithmetic without affecting flags + sa, + /// Store AH into flags + sahf, + /// Integer subtraction with borrow + sbb, + /// Scan string + sca, + /// Send user interprocessor interrupt + senduipi, + /// Serialize instruction execution + serialize, + /// Set byte on condition + set, + /// Logical shift left + /// Double precision shift left + /// Logical shift right + /// Double precision shift right + /// Shift left logical without affecting flags + /// Shift right logical without affecting flags + sh, + /// Store interrupt descriptor table register + sidt, + /// Store local descriptor table register + sldt, + /// Store machine status word + smsw, + /// Subtract + /// Subtract packed integers + /// Subtract packed single-precision floating-point values + /// Subtract scalar single-precision floating-point values + /// Subtract packed double-precision floating-point values + /// Subtract scalar double-precision floating-point values + /// Packed single-precision floating-point horizontal subtract + /// Packed double-precision floating-point horizontal subtract + /// Packed horizontal subtract + /// Packed horizontal subtract and saturate + sub, + /// Set carry flag + /// Set direction flag + /// Set interrupt flag + /// Store binary coded decimal integer and pop + /// Store floating-point value + /// Store integer + /// Store x87 FPU control word + /// Store x87 FPU environment + /// Store x87 FPU status word + /// Store MXCSR register state + st, + /// Store string + sto, + /// Test condition + /// Logical compare + /// Packed bit test + @"test", + /// Undefined instruction + ud, + /// User level set up monitor address + umonitor, + /// Verify a segment for reading + /// Verify a segment for writing + ver, + /// Write to model specific register + /// Write to model specific register + /// Write to model specific register + /// Write to shadow stack + /// Write to user shadow stack + wr, + /// Exchange and add + xadd, + /// Exchange register/memory with register + /// Exchange register contents + xch, + /// Get value of extended control register + xgetbv, + /// Table look-up translation + xlat, + /// Logical exclusive-or + /// Bitwise logical xor of packed single-precision floating-point values + /// Bitwise logical xor of packed double-precision floating-point values + xor, + + // X87 + /// Compute 2^x-1 + @"2xm1", + /// Absolute value + abs, + /// Change sign + chs, + /// Clear exceptions + clex, + /// Compare floating-point values + com, + /// Compare floating-point values and set EFLAGS + /// Compare scalar ordered single-precision floating-point values + /// Compare scalar ordered double-precision floating-point values + comi, + /// Cosine + cos, + /// Reverse divide + divr, + /// Free floating-point register + free, + /// Initialize floating-point unit + init, + /// Load binary coded decimal integer + /// Load floating-point value + /// Load integer + /// Load constant + /// Load x87 FPU control word + /// Load x87 FPU environment + /// Load MXCSR register state + ld, + /// Partial arctangent + patan, + /// Partial remainder + prem, + /// Partial tangent + ptan, + /// Round to integer + rndint, + /// Restore x87 FPU state + /// Restore x87 FPU, MMX, XMM, and MXCSR state + rstor, + /// Store x87 FPU state + /// Save x87 FPU, MMX technology, and MXCSR state + save, + /// Scale + scale, + /// Sine + sin, + /// Sine and cosine + sincos, + /// Square root + /// Square root of packed single-precision floating-point values + /// Square root of scalar single-precision floating-point value + /// Square root of packed double-precision floating-point values + /// Square root of scalar double-precision floating-point value + sqrt, + /// Store integer with truncation + stt, + /// Reverse subtract + subr, + /// Test + tst, + /// Unordered compare floating-point values + ucom, + /// Unordered compare floating-point values and set EFLAGS + /// Unordered compare scalar single-precision floating-point values + /// Unordered compare scalar double-precision floating-point values + ucomi, + /// Wait + /// User level monitor wait + wait, + /// Examine floating-point + xam, + /// Extract exponent and significand + xtract, + /// Compute y * log2x + /// Compute y * log2(x + 1) + yl2x, + + // MMX + /// Pack with signed saturation + ackssw, + /// Pack with signed saturation + ackssd, + /// Pack with unsigned saturation + ackusw, + /// Add packed signed integers with signed saturation + adds, + /// Add packed unsigned integers with unsigned saturation + addus, + /// Logical and not + /// Bitwise logical and not of packed single-precision floating-point values + /// Bitwise logical and not of packed double-precision floating-point values + andn, + /// Compare packed data for equal + cmpeq, + /// Compare packed data for greater than + cmpgt, + /// Empty MMX technology state + emms, + /// Multiply and add packed signed and unsigned bytes + maddubs, + /// Multiply and add packed integers + maddw, + /// Multiply packed signed integers and store low result + mull, + /// Multiply packed signed integers and store high result + mulh, + /// Shift packed data left logical + sll, + /// Shift packed data right arithmetic + sra, + /// Shift packed data right logical + srl, + /// Subtract packed signed integers with signed saturation + subs, + /// Subtract packed unsigned integers with unsigned saturation + subus, + /// Unpack high data + unpckhbw, + /// Unpack high data + unpckhdq, + /// Unpack high data + unpckhwd, + /// Unpack low data + unpcklbw, + /// Unpack low data + unpckldq, + /// Unpack low data + unpcklwd, + + // SSE + /// Average packed integers + avg, + /// Convert packed doubleword integers to packed single-precision floating-point values + /// Convert packed doubleword integers to packed double-precision floating-point values + cvtpi2, + /// Convert packed single-precision floating-point values to packed doubleword integers + cvtps2pi, + /// Convert doubleword integer to scalar single-precision floating-point value + /// Convert doubleword integer to scalar double-precision floating-point value + cvtsi2, + /// Convert scalar single-precision floating-point value to doubleword integer + cvtss2si, + /// Convert with truncation packed single-precision floating-point values to packed doubleword integers + cvttps2pi, + /// Convert with truncation scalar single-precision floating-point value to doubleword integer + cvttss2si, + /// Extract byte + /// Extract word + /// Extract doubleword + /// Extract quadword + extr, + /// Insert byte + /// Insert word + /// Insert doubleword + /// Insert quadword + insr, + /// Maximum of packed single-precision floating-point values + /// Maximum of scalar single-precision floating-point values + /// Maximum of packed double-precision floating-point values + /// Maximum of scalar double-precision floating-point values + max, + /// Maximum of packed signed integers + maxs, + /// Maximum of packed unsigned integers + maxu, + /// Minimum of packed single-precision floating-point values + /// Minimum of scalar single-precision floating-point values + /// Minimum of packed double-precision floating-point values + /// Minimum of scalar double-precision floating-point values + min, + /// Minimum of packed signed integers + mins, + /// Minimum of packed unsigned integers + minu, + /// Move aligned packed single-precision floating-point values + /// Move aligned packed double-precision floating-point values + mova, + /// Move high packed single-precision floating-point values + /// Move high packed double-precision floating-point values + movh, + /// Move packed single-precision floating-point values high to low + movhl, + /// Move low packed single-precision floating-point values + /// Move low packed double-precision floating-point values + movl, + /// Move packed single-precision floating-point values low to high + movlh, + /// Move byte mask + /// Extract packed single precision floating-point sign mask + /// Extract packed double precision floating-point sign mask + movmsk, + /// Move unaligned packed single-precision floating-point values + /// Move unaligned packed double-precision floating-point values + movu, + /// Multiply packed unsigned integers and store high result + mulhu, + /// Prefetch data into caches + /// Prefetch data into caches with intent to write + prefetch, + /// Compute sum of absolute differences + sadb, + /// Packed interleave shuffle of quadruplets of single-precision floating-point values + /// Packed interleave shuffle of pairs of double-precision floating-point values + /// Shuffle packed doublewords + /// Shuffle packed words + shuf, + /// Unpack and interleave high packed single-precision floating-point values + /// Unpack and interleave high packed double-precision floating-point values + unpckh, + /// Unpack and interleave low packed single-precision floating-point values + /// Unpack and interleave low packed double-precision floating-point values + unpckl, + + // SSE2 + /// Convert packed doubleword integers to packed single-precision floating-point values + /// Convert packed doubleword integers to packed double-precision floating-point values + cvtdq2, + /// Convert packed double-precision floating-point values to packed doubleword integers + cvtpd2dq, + /// Convert packed double-precision floating-point values to packed doubleword integers + cvtpd2pi, + /// Convert packed double-precision floating-point values to packed single-precision floating-point values + cvtpd2, + /// Convert packed single-precision floating-point values to packed doubleword integers + cvtps2dq, + /// Convert packed single-precision floating-point values to packed double-precision floating-point values + cvtps2, + /// Convert scalar double-precision floating-point value to doubleword integer + cvtsd2si, + /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value + cvtsd2, + /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value + cvtss2, + /// Convert with truncation packed double-precision floating-point values to packed doubleword integers + cvttpd2dq, + /// Convert with truncation packed double-precision floating-point values to packed doubleword integers + cvttpd2pi, + /// Convert with truncation packed single-precision floating-point values to packed doubleword integers + cvttps2dq, + /// Convert with truncation scalar double-precision floating-point value to doubleword integer + cvttsd2si, + /// Galois field affine transformation inverse + gf2p8affineinvq, + /// Galois field affine transformation + gf2p8affineq, + /// Galois field multiply bytes + gf2p8mul, + /// Shuffle packed high words + shufh, + /// Shuffle packed low words + shufl, + /// Unpack high data + unpckhqdq, + /// Unpack low data + unpcklqdq, + + // SSE3 + /// Packed single-precision floating-point add/subtract + /// Packed double-precision floating-point add/subtract + addsub, + /// Replicate double floating-point values + movddup, + /// Replicate single floating-point values + movshdup, + /// Replicate single floating-point values + movsldup, + + // SSSE3 + /// Packed align right + alignr, + /// Packed multiply high with round and scale + mulhrs, + /// Packed sign + sign, + + // SSE4.1 + /// Pack with unsigned saturation + ackusd, + /// Blend packed single-precision floating-point values + /// Blend scalar single-precision floating-point values + /// Blend packed double-precision floating-point values + /// Blend scalar double-precision floating-point values + /// Blend packed dwords + blend, + /// Variable blend packed single-precision floating-point values + /// Variable blend scalar single-precision floating-point values + /// Variable blend packed double-precision floating-point values + /// Variable blend scalar double-precision floating-point values + blendv, + /// Dot product of packed single-precision floating-point values + /// Dot product of packed double-precision floating-point values + dp, + /// Extract packed floating-point values + /// Extract packed integer values + extract, + /// Insert scalar single-precision floating-point value + /// Insert packed floating-point values + insert, + /// Packed horizontal word minimum + minposu, + /// Packed move with sign extend + movsxb, + movsxd, + movsxw, + /// Packed move with zero extend + movzxb, + movzxd, + movzxw, + /// Round packed single-precision floating-point values + /// Round scalar single-precision floating-point value + /// Round packed double-precision floating-point values + /// Round scalar double-precision floating-point value + round, + + // SSE4.2 + /// Accumulate CRC32 value + crc32, + + // AES + /// Perform one round of an AES decryption flow + /// Perform ten rounds of AES decryption flow with key locker using 128-bit key + /// Perform ten rounds of AES decryption flow with key locker using 256-bit key + /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 128-bit key + /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 256-bit key + aesdec, + /// Perform last round of an AES decryption flow + aesdeclast, + /// Perform one round of an AES encryption flow + /// Perform ten rounds of AES encryption flow with key locker using 128-bit key + /// Perform ten rounds of AES encryption flow with key locker using 256-bit key + /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 128-bit key + /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 256-bit key + aesenc, + /// Perform last round of an AES encryption flow + aesenclast, + /// Perform the AES InvMixColumn transformation + aesimc, + /// AES round key generation assist + aeskeygenassist, + + // SHA + /// Perform four rounds of SHA1 operation + sha1rnds, + /// Calculate SHA1 state variable E after four rounds + sha1nexte, + /// Perform an intermediate calculation for the next four SHA1 message dwords + /// Perform a final calculation for the next four SHA1 message dwords + sha1msg, + /// Perform an intermediate calculation for the next four SHA256 message dwords + /// Perform a final calculation for the next four SHA256 message dwords + sha256msg, + /// Perform two rounds of SHA256 operation + sha256rnds, + + // AVX + /// Load with broadcast floating-point data + /// Load integer and broadcast + broadcast, + /// Conditional SIMD packed loads and stores + /// Condition SIMD integer packed loads and stores + maskmov, + /// Permute floating-point values + /// Permute integer values + perm2, + /// Permute in-lane pairs of double-precision floating-point values + /// Permute in-lane quadruples of single-precision floating-point values + permil, + + // BMI + /// Bit field extract + bextr, + /// Extract lowest set isolated bit + /// Get mask up to lowest set bit + /// Reset lowest set bit + bls, + /// Count the number of trailing zero bits + tzcnt, + + // BMI2 + /// Zero high bits starting with specified bit position + bzhi, + /// Parallel bits deposit + pdep, + /// Parallel bits extract + pext, + + // F16C + /// Convert 16-bit floating-point values to single-precision floating-point values + cvtph2, + /// Convert single-precision floating-point values to 16-bit floating-point values + cvtps2ph, + + // FMA + /// Fused multiply-add of packed single-precision floating-point values + /// Fused multiply-add of scalar single-precision floating-point values + /// Fused multiply-add of packed double-precision floating-point values + /// Fused multiply-add of scalar double-precision floating-point values + fmadd132, + /// Fused multiply-add of packed single-precision floating-point values + /// Fused multiply-add of scalar single-precision floating-point values + /// Fused multiply-add of packed double-precision floating-point values + /// Fused multiply-add of scalar double-precision floating-point values + fmadd213, + /// Fused multiply-add of packed single-precision floating-point values + /// Fused multiply-add of scalar single-precision floating-point values + /// Fused multiply-add of packed double-precision floating-point values + /// Fused multiply-add of scalar double-precision floating-point values + fmadd231, + + // AVX2 + /// Permute packed doubleword elements + /// Permute packed qword elements + /// Permute double-precision floating-point elements + /// Permute single-precision floating-point elements + perm, + /// Variable bit shift left logical + sllv, + /// Variable bit shift right arithmetic + srav, + /// Variable bit shift right logical + srlv, + + // ADX + /// Unsigned integer addition of two operands with overflow flag + ado, + + // AESKLE + /// Encode 128-bit key with key locker + /// Encode 256-bit key with key locker + encodekey, + /// Load internal wrapping key with key locker + loadiwkey, + + /// A pseudo instruction that requires special lowering. + /// This should be the only tag in this enum that doesn't + /// directly correspond to one or more instruction mnemonics. + pseudo, + }; + + pub const FixedTag = struct { Fixes, Tag }; + + pub const Ops = enum(u8) { + /// No data associated with this instruction (only mnemonic is used). + none, + /// Single register operand. + /// Uses `r` payload. + r, + /// Register, register operands. + /// Uses `rr` payload. + rr, + /// Register, register, register operands. + /// Uses `rrr` payload. + rrr, + /// Register, register, register, register operands. + /// Uses `rrrr` payload. + rrrr, + /// Register, register, register, immediate (byte) operands. + /// Uses `rrri` payload. + rrri, + /// Register, register, immediate (sign-extended) operands. + /// Uses `rri` payload. + rri_s, + /// Register, register, immediate (unsigned) operands. + /// Uses `rri` payload. + rri_u, + /// Register, immediate (sign-extended) operands. + /// Uses `ri` payload. + ri_s, + /// Register, immediate (unsigned) operands. + /// Uses `ri` payload. + ri_u, + /// Register, 64-bit unsigned immediate operands. + /// Uses `ri` payload with `i` index of extra data of type `Imm64`. + ri_64, + /// Immediate (sign-extended) operand. + /// Uses `i` payload. + i_s, + /// Immediate (unsigned) operand. + /// Uses `i` payload. + i_u, + /// Immediate (word), immediate (byte) operands. + /// Uses `ii` payload. + ii, + /// Immediate (byte), register operands. + /// Uses `ri` payload. + ir, + /// Register, memory operands. + /// Uses `rx` payload with extra data of type `Memory`. + rm, + /// Register, memory, register operands. + /// Uses `rrx` payload with extra data of type `Memory`. + rmr, + /// Register, memory, immediate (word) operands. + /// Uses `rix` payload with extra data of type `Memory`. + rmi, + /// Register, memory, immediate (signed) operands. + /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`. + rmi_s, + /// Register, memory, immediate (unsigned) operands. + /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`. + rmi_u, + /// Register, register, memory. + /// Uses `rrix` payload with extra data of type `Memory`. + rrm, + /// Register, register, memory, register. + /// Uses `rrrx` payload with extra data of type `Memory`. + rrmr, + /// Register, register, memory, immediate (byte) operands. + /// Uses `rrix` payload with extra data of type `Memory`. + rrmi, + /// Single memory operand. + /// Uses `x` payload with extra data of type `Memory`. + m, + /// Memory, immediate (sign-extend) operands. + /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`. + mi_s, + /// Memory, immediate (unsigned) operands. + /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`. + mi_u, + /// Memory, register operands. + /// Uses `rx` payload with extra data of type `Memory`. + mr, + /// Memory, register, register operands. + /// Uses `rrx` payload with extra data of type `Memory`. + mrr, + /// Memory, register, immediate (word) operands. + /// Uses `rix` payload with extra data of type `Memory`. + mri, + /// References another Mir instruction directly. + /// Uses `inst` payload. + inst, + /// References a nav. + /// Uses `nav` payload. + nav, + /// References an uav. + /// Uses `uav` payload. + uav, + /// References a lazy symbol. + /// Uses `lazy_sym` payload. + lazy_sym, + /// References an external symbol. + /// Uses `extern_func` payload. + extern_func, + + // Pseudo instructions: + + /// Conditional move if zero flag set and parity flag not set + /// Clobbers the source operand! + /// Uses `rr` payload. + pseudo_cmov_z_and_np_rr, + /// Conditional move if zero flag not set or parity flag set + /// Uses `rr` payload. + pseudo_cmov_nz_or_p_rr, + /// Conditional move if zero flag not set or parity flag set + /// Uses `rx` payload. + pseudo_cmov_nz_or_p_rm, + /// Set byte if zero flag set and parity flag not set + /// Requires a scratch register! + /// Uses `rr` payload. + pseudo_set_z_and_np_r, + /// Set byte if zero flag set and parity flag not set + /// Requires a scratch register! + /// Uses `rx` payload. + pseudo_set_z_and_np_m, + /// Set byte if zero flag not set or parity flag set + /// Requires a scratch register! + /// Uses `rr` payload. + pseudo_set_nz_or_p_r, + /// Set byte if zero flag not set or parity flag set + /// Requires a scratch register! + /// Uses `rx` payload. + pseudo_set_nz_or_p_m, + /// Jump if zero flag set and parity flag not set + /// Uses `inst` payload. + pseudo_j_z_and_np_inst, + /// Jump if zero flag not set or parity flag set + /// Uses `inst` payload. + pseudo_j_nz_or_p_inst, + + /// Probe alignment + /// Uses `ri` payload. + pseudo_probe_align_ri_s, + /// Probe adjust unrolled + /// Uses `ri` payload. + pseudo_probe_adjust_unrolled_ri_s, + /// Probe adjust setup + /// Uses `rri` payload. + pseudo_probe_adjust_setup_rri_s, + /// Probe adjust loop + /// Uses `rr` payload. + pseudo_probe_adjust_loop_rr, + + /// Push registers + /// Uses `reg_list` payload. + pseudo_push_reg_list, + /// Pop registers + /// Uses `reg_list` payload. + pseudo_pop_reg_list, + + /// Define cfa rule as offset from register. + /// Uses `ri` payload. + pseudo_cfi_def_cfa_ri_s, + /// Modify cfa rule register. + /// Uses `r` payload. + pseudo_cfi_def_cfa_register_r, + /// Modify cfa rule offset. + /// Uses `i` payload. + pseudo_cfi_def_cfa_offset_i_s, + /// Offset cfa rule offset. + /// Uses `i` payload. + pseudo_cfi_adjust_cfa_offset_i_s, + /// Define register rule as stored at offset from cfa. + /// Uses `ri` payload. + pseudo_cfi_offset_ri_s, + /// Define register rule as offset from cfa. + /// Uses `ri` payload. + pseudo_cfi_val_offset_ri_s, + /// Define register rule as stored at offset from cfa rule register. + /// Uses `ri` payload. + pseudo_cfi_rel_offset_ri_s, + /// Define register rule as register. + /// Uses `rr` payload. + pseudo_cfi_register_rr, + /// Define register rule from initial. + /// Uses `r` payload. + pseudo_cfi_restore_r, + /// Define register rule as undefined. + /// Uses `r` payload. + pseudo_cfi_undefined_r, + /// Define register rule as itself. + /// Uses `r` payload. + pseudo_cfi_same_value_r, + /// Push cfi state. + pseudo_cfi_remember_state_none, + /// Pop cfi state. + pseudo_cfi_restore_state_none, + /// Raw cfi bytes. + /// Uses `bytes` payload. + pseudo_cfi_escape_bytes, + + /// End of prologue + /// Uses `none` payload. + pseudo_dbg_prologue_end_none, + /// Update debug line with is_stmt register set + /// Uses `line_column` payload. + pseudo_dbg_line_stmt_line_column, + /// Update debug line with is_stmt register clear + /// Uses `line_column` payload. + pseudo_dbg_line_line_column, + /// Start of epilogue + /// Uses `none` payload. + pseudo_dbg_epilogue_begin_none, + /// Start of lexical block + /// Uses `none` payload. + pseudo_dbg_enter_block_none, + /// End of lexical block + /// Uses `none` payload. + pseudo_dbg_leave_block_none, + /// Start of inline function + /// Uses `ip_index` payload. + pseudo_dbg_enter_inline_func, + /// End of inline function + /// Uses `ip_index` payload. + pseudo_dbg_leave_inline_func, + /// Local argument. + /// Uses `none` payload. + pseudo_dbg_arg_none, + /// Local argument. + /// Uses `i` payload. + pseudo_dbg_arg_i_s, + /// Local argument. + /// Uses `i` payload. + pseudo_dbg_arg_i_u, + /// Local argument. + /// Uses `i64` payload. + pseudo_dbg_arg_i_64, + /// Local argument. + /// Uses `ro` payload. + pseudo_dbg_arg_ro, + /// Local argument. + /// Uses `fa` payload. + pseudo_dbg_arg_fa, + /// Local argument. + /// Uses `x` payload with extra data of type `Memory`. + pseudo_dbg_arg_m, + /// Local argument. + /// Uses `ip_index` payload. + pseudo_dbg_arg_val, + /// Remaining arguments are varargs. + pseudo_dbg_var_args_none, + /// Local variable. + /// Uses `none` payload. + pseudo_dbg_var_none, + /// Local variable. + /// Uses `i` payload. + pseudo_dbg_var_i_s, + /// Local variable. + /// Uses `i` payload. + pseudo_dbg_var_i_u, + /// Local variable. + /// Uses `i64` payload. + pseudo_dbg_var_i_64, + /// Local variable. + /// Uses `ro` payload. + pseudo_dbg_var_ro, + /// Local variable. + /// Uses `fa` payload. + pseudo_dbg_var_fa, + /// Local variable. + /// Uses `x` payload with extra data of type `Memory`. + pseudo_dbg_var_m, + /// Local variable. + /// Uses `ip_index` payload. + pseudo_dbg_var_val, + + /// Tombstone + /// Emitter should skip this instruction. + pseudo_dead_none, + }; + + pub const Data = union { + none: struct { + fixes: Fixes = ._, + }, + /// References another Mir instruction. + inst: struct { + fixes: Fixes = ._, + inst: Index, + }, + /// A 32-bit immediate value. + i64: u64, + i: struct { + fixes: Fixes = ._, + i: u32, + }, + ii: struct { + fixes: Fixes = ._, + i1: u16, + i2: u8, + }, + r: struct { + fixes: Fixes = ._, + r1: Register, + }, + rr: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + }, + rrr: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + r3: Register, + }, + rrrr: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + r3: Register, + r4: Register, + }, + rrri: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + r3: Register, + i: u8, + }, + rri: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + i: u32, + }, + /// Register, immediate. + ri: struct { + fixes: Fixes = ._, + r1: Register, + i: u32, + }, + /// Register, followed by custom payload found in extra. + rx: struct { + fixes: Fixes = ._, + r1: Register, + payload: u32, + }, + /// Register, register, followed by Custom payload found in extra. + rrx: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + payload: u32, + }, + /// Register, register, register, followed by Custom payload found in extra. + rrrx: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + r3: Register, + payload: u32, + }, + /// Register, byte immediate, followed by Custom payload found in extra. + rix: struct { + fixes: Fixes = ._, + r1: Register, + i: u16, + payload: u32, + }, + /// Register, register, byte immediate, followed by Custom payload found in extra. + rrix: struct { + fixes: Fixes = ._, + r1: Register, + r2: Register, + i: u8, + payload: u32, + }, + /// Custom payload found in extra. + x: struct { + fixes: Fixes = ._, + payload: u32, + }, + bytes: struct { + payload: u32, + len: u32, + + pub fn get(bytes: @This(), mir: Mir) []const u8 { + return std.mem.sliceAsBytes(mir.extra[bytes.payload..])[0..bytes.len]; + } + }, + fa: bits.FrameAddr, + ro: bits.RegisterOffset, + nav: bits.NavOffset, + uav: InternPool.Key.Ptr.BaseAddr.Uav, + lazy_sym: link.File.LazySymbol, + extern_func: Mir.NullTerminatedString, + /// Debug line and column position + line_column: struct { + line: u32, + column: u32, + }, + ip_index: InternPool.Index, + /// Register list + reg_list: RegisterList, + }; + + comptime { + if (!std.debug.runtime_safety) { + // Make sure we don't accidentally make instructions bigger than expected. + // Note that in safety builds, Zig is allowed to insert a secret field for safety checks. + assert(@sizeOf(Data) == 8); + } + const Mnemonic = @import("Encoding.zig").Mnemonic; + if (@typeInfo(Mnemonic).@"enum".fields.len != 977 or + @typeInfo(Fixes).@"enum".fields.len != 231 or + @typeInfo(Tag).@"enum".fields.len != 251) + { + const cond_src = (struct { + fn src() std.builtin.SourceLocation { + return @src(); + } + }).src(); + @setEvalBranchQuota(1_750_000); + for (@typeInfo(Mnemonic).@"enum".fields) |mnemonic| { + if (mnemonic.name[0] == '.') continue; + for (@typeInfo(Fixes).@"enum".fields) |fixes| { + const pattern = fixes.name[if (std.mem.indexOfScalar(u8, fixes.name, ' ')) |index| index + " ".len else 0..]; + const wildcard_index = std.mem.indexOfScalar(u8, pattern, '_').?; + const mnem_prefix = pattern[0..wildcard_index]; + const mnem_suffix = pattern[wildcard_index + "_".len ..]; + if (!std.mem.startsWith(u8, mnemonic.name, mnem_prefix)) continue; + if (!std.mem.endsWith(u8, mnemonic.name, mnem_suffix)) continue; + if (@hasField( + Tag, + mnemonic.name[mnem_prefix.len .. mnemonic.name.len - mnem_suffix.len], + )) break; + } else @compileError("'" ++ mnemonic.name ++ "' is not encodable in Mir"); + } + @compileError(std.fmt.comptimePrint( + \\All mnemonics are encodable in Mir! You may now change the condition at {s}:{d} to: + \\if (@typeInfo(Mnemonic).@"enum".fields.len != {d} or + \\ @typeInfo(Fixes).@"enum".fields.len != {d} or + \\ @typeInfo(Tag).@"enum".fields.len != {d}) + , .{ + cond_src.file, + cond_src.line - 6, + @typeInfo(Mnemonic).@"enum".fields.len, + @typeInfo(Fixes).@"enum".fields.len, + @typeInfo(Tag).@"enum".fields.len, + })); + } + } +}; + +/// Used in conjunction with payload to transfer a list of used registers in a compact manner. +pub const RegisterList = struct { + bitset: BitSet, + + const BitSet = std.bit_set.IntegerBitSet(32); + const Self = @This(); + + pub const empty: RegisterList = .{ .bitset = .initEmpty() }; + + fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt { + for (registers, 0..) |cpreg, i| { + if (reg.id() == cpreg.id()) return @intCast(i); + } + unreachable; // register not in input register list! + } + + pub fn push(self: *Self, registers: []const Register, reg: Register) void { + const index = getIndexForReg(registers, reg); + self.bitset.set(index); + } + + pub fn isSet(self: Self, registers: []const Register, reg: Register) bool { + const index = getIndexForReg(registers, reg); + return self.bitset.isSet(index); + } + + pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) { + return self.bitset.iterator(options); + } + + pub fn count(self: Self) i32 { + return @intCast(self.bitset.count()); + } + + pub fn size(self: Self, target: *const std.Target) i32 { + return @intCast(self.bitset.count() * @as(u4, switch (target.cpu.arch) { + else => unreachable, + .x86 => 4, + .x86_64 => 8, + })); + } +}; + +pub const NullTerminatedString = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn toSlice(nts: NullTerminatedString, mir: *const Mir) ?[:0]const u8 { + if (nts == .none) return null; + const string_bytes = mir.string_bytes[@intFromEnum(nts)..]; + return string_bytes[0..std.mem.indexOfScalar(u8, string_bytes, 0).? :0]; + } +}; + +pub const Local = struct { + name: NullTerminatedString, + type: InternPool.Index, +}; + +pub const Imm32 = struct { + imm: u32, +}; + +pub const Imm64 = struct { + msb: u32, + lsb: u32, + + pub fn encode(v: u64) Imm64 { + return .{ + .msb = @truncate(v >> 32), + .lsb = @truncate(v), + }; + } + + pub fn decode(imm: Imm64) u64 { + var res: u64 = 0; + res |= @as(u64, @intCast(imm.msb)) << 32; + res |= @as(u64, @intCast(imm.lsb)); + return res; + } +}; + +pub const Memory = struct { + info: Info, + base: u32, + off: u32, + extra: u32, + + pub const Info = packed struct(u32) { + base: @typeInfo(bits.Memory.Base).@"union".tag_type.?, + mod: @typeInfo(bits.Memory.Mod).@"union".tag_type.?, + size: bits.Memory.Size, + index: Register, + scale: bits.Memory.Scale, + _: u13 = undefined, + }; + + pub fn encode(mem: bits.Memory) Memory { + return .{ + .info = .{ + .base = mem.base, + .mod = mem.mod, + .size = switch (mem.mod) { + .rm => |rm| rm.size, + .off => undefined, + }, + .index = switch (mem.mod) { + .rm => |rm| rm.index, + .off => undefined, + }, + .scale = switch (mem.mod) { + .rm => |rm| rm.scale, + .off => undefined, + }, + }, + .base = switch (mem.base) { + .none, .table => undefined, + .reg => |reg| @intFromEnum(reg), + .frame => |frame_index| @intFromEnum(frame_index), + .rip_inst => |inst_index| inst_index, + .nav => |nav| @intFromEnum(nav), + .uav => |uav| @intFromEnum(uav.val), + .lazy_sym => |lazy_sym| @intFromEnum(lazy_sym.ty), + .extern_func => |extern_func| @intFromEnum(extern_func), + }, + .off = switch (mem.mod) { + .rm => |rm| @bitCast(rm.disp), + .off => |off| @truncate(off), + }, + .extra = switch (mem.mod) { + .rm => switch (mem.base) { + else => undefined, + .uav => |uav| @intFromEnum(uav.orig_ty), + .lazy_sym => |lazy_sym| @intFromEnum(lazy_sym.kind), + }, + .off => switch (mem.base) { + .reg => @intCast(mem.mod.off >> 32), + else => unreachable, + }, + }, + }; + } + + pub fn decode(mem: Memory) encoder.Instruction.Memory { + switch (mem.info.mod) { + .rm => { + if (mem.info.base == .reg and @as(Register, @enumFromInt(mem.base)) == .rip) { + assert(mem.info.index == .none and mem.info.scale == .@"1"); + return encoder.Instruction.Memory.initRip(mem.info.size, @bitCast(mem.off)); + } + return encoder.Instruction.Memory.initSib(mem.info.size, .{ + .disp = @bitCast(mem.off), + .base = switch (mem.info.base) { + .none => .none, + .reg => .{ .reg = @enumFromInt(mem.base) }, + .frame => .{ .frame = @enumFromInt(mem.base) }, + .table => .table, + .rip_inst => .{ .rip_inst = mem.base }, + .nav => .{ .nav = @enumFromInt(mem.base) }, + .uav => .{ .uav = .{ .val = @enumFromInt(mem.base), .orig_ty = @enumFromInt(mem.extra) } }, + .lazy_sym => .{ .lazy_sym = .{ .kind = @enumFromInt(mem.extra), .ty = @enumFromInt(mem.base) } }, + .extern_func => .{ .extern_func = @enumFromInt(mem.base) }, + }, + .scale_index = switch (mem.info.index) { + .none => null, + else => |index| .{ .scale = switch (mem.info.scale) { + inline else => |scale| comptime std.fmt.parseInt( + u4, + @tagName(scale), + 10, + ) catch unreachable, + }, .index = index }, + }, + }); + }, + .off => { + assert(mem.info.base == .reg); + return encoder.Instruction.Memory.initMoffs( + @enumFromInt(mem.base), + @as(u64, mem.extra) << 32 | mem.off, + ); + }, + } + } +}; + +pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { + mir.instructions.deinit(gpa); + gpa.free(mir.extra); + gpa.free(mir.string_bytes); + gpa.free(mir.locals); + gpa.free(mir.table); + gpa.free(mir.memoized_encodings); + mir.frame_locs.deinit(gpa); + mir.* = undefined; +} + +pub fn emit( + mir: Mir, + lf: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + func_index: InternPool.Index, + atom_index: u32, + w: *std.Io.Writer, + debug_output: link.File.DebugInfoOutput, +) codegen.CodeGenError!void { + const zcu = pt.zcu; + const comp = zcu.comp; + const gpa = comp.gpa; + const func = zcu.funcInfo(func_index); + const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?; + const nav = func.owner_nav; + const mod = zcu.navFileScope(nav).mod.?; + var e: Emit = .{ + .lower = .{ + .target = &mod.resolved_target.result, + .allocator = gpa, + .mir = mir, + .cc = fn_info.cc, + .src_loc = src_loc, + }, + .bin_file = lf, + .pt = pt, + .pic = mod.pic, + .atom_index = atom_index, + .debug_output = debug_output, + .w = w, + + .prev_di_loc = .{ + .line = func.lbrace_line, + .column = func.lbrace_column, + .is_stmt = switch (debug_output) { + .dwarf => |dwarf| dwarf.dwarf.debug_line.header.default_is_stmt, + .none => undefined, + }, + }, + .prev_di_pc = 0, + + .code_offset_mapping = .empty, + .relocs = .empty, + .table_relocs = .empty, + }; + defer e.deinit(); + e.emitMir() catch |err| switch (err) { + error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, e.lower.err_msg.?), + error.InvalidInstruction, error.CannotEncode => return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}), + else => return zcu.codegenFail(nav, "emit MIR failed: {s}", .{@errorName(err)}), + }; +} + +pub fn emitLazy( + mir: Mir, + lf: *link.File, + pt: Zcu.PerThread, + src_loc: Zcu.LazySrcLoc, + lazy_sym: link.File.LazySymbol, + atom_index: u32, + w: *std.Io.Writer, + debug_output: link.File.DebugInfoOutput, +) codegen.CodeGenError!void { + const zcu = pt.zcu; + const comp = zcu.comp; + const gpa = comp.gpa; + const mod = comp.root_mod; + var e: Emit = .{ + .lower = .{ + .target = &mod.resolved_target.result, + .allocator = gpa, + .mir = mir, + .cc = .auto, + .src_loc = src_loc, + }, + .bin_file = lf, + .pt = pt, + .pic = mod.pic, + .atom_index = atom_index, + .debug_output = debug_output, + .w = w, + + .prev_di_loc = undefined, + .prev_di_pc = undefined, + + .code_offset_mapping = .empty, + .relocs = .empty, + .table_relocs = .empty, + }; + defer e.deinit(); + e.emitMir() catch |err| switch (err) { + error.LowerFail, error.EmitFail => return zcu.codegenFailTypeMsg(lazy_sym.ty, e.lower.err_msg.?), + error.InvalidInstruction, error.CannotEncode => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}), + else => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s}", .{@errorName(err)}), + }; +} + +pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end: u32 } { + const fields = std.meta.fields(T); + var i: u32 = index; + var result: T = undefined; + inline for (fields) |field| { + @field(result, field.name) = switch (field.type) { + u32 => mir.extra[i], + i32, Memory.Info => @bitCast(mir.extra[i]), + bits.FrameIndex => @enumFromInt(mir.extra[i]), + else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)), + }; + i += 1; + } + return .{ + .data = result, + .end = i, + }; +} + +pub const FrameLoc = struct { + base: Register, + disp: i32, +}; + +pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffset { + const frame_loc = mir.frame_locs.get(@intFromEnum(frame_addr.index)); + return .{ .reg = frame_loc.base, .off = frame_loc.disp + frame_addr.off }; +} + +pub fn resolveMemoryExtra(mir: Mir, payload: u32) Memory { + const mem = mir.extraData(Mir.Memory, payload).data; + return switch (mem.info.base) { + .none, .reg, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => mem, + .frame => if (mir.frame_locs.len > 0) .{ + .info = .{ + .base = .reg, + .mod = mem.info.mod, + .size = mem.info.size, + .index = mem.info.index, + .scale = mem.info.scale, + }, + .base = @intFromEnum(mir.frame_locs.items(.base)[mem.base]), + .off = @bitCast(mir.frame_locs.items(.disp)[mem.base] + @as(i32, @bitCast(mem.off))), + .extra = mem.extra, + } else mem, + }; +} + +const assert = std.debug.assert; +const bits = @import("bits.zig"); +const builtin = @import("builtin"); +const encoder = @import("encoder.zig"); +const std = @import("std"); + +const InternPool = @import("../../InternPool.zig"); +const Mir = @This(); +const Register = bits.Register; +const Emit = @import("Emit.zig"); +const codegen = @import("../../codegen.zig"); +const link = @import("../../link.zig"); +const Zcu = @import("../../Zcu.zig"); diff --git a/src/codegen/x86_64/abi.zig b/src/codegen/x86_64/abi.zig new file mode 100644 index 0000000000000000000000000000000000000000..041eb1632d3b91bb5a9e47d8a1d800c2682f6b6b --- /dev/null +++ b/src/codegen/x86_64/abi.zig @@ -0,0 +1,608 @@ +pub const Class = enum { + /// INTEGER: This class consists of integral types that fit into one of the general + /// purpose registers. + integer, + /// SSE: The class consists of types that fit into a vector register. + sse, + /// SSEUP: The class consists of types that fit into a vector register and can be passed + /// and returned in the upper bytes of it. + sseup, + /// X87, X87UP: These classes consist of types that will be returned via the + /// x87 FPU. + x87, + /// The 15-bit exponent, 1-bit sign, and 6 bytes of padding of an `f80`. + x87up, + /// NO_CLASS: This class is used as initializer in the algorithms. It will be used for + /// padding and empty structures and unions. + none, + /// MEMORY: This class consists of types that will be passed and returned in mem- + /// ory via the stack. + memory, + /// Win64 passes 128-bit integers as `Class.memory` but returns them as `Class.sse`. + win_i128, + /// A `Class.sse` containing one `f32`. + float, + /// A `Class.sse` containing two `f32`s. + float_combine, + /// Clang passes each vector element in a separate `Class.integer`, but returns as `Class.memory`. + integer_per_element, + + pub const one_integer: [8]Class = .{ + .integer, .none, .none, .none, + .none, .none, .none, .none, + }; + pub const two_integers: [8]Class = .{ + .integer, .integer, .none, .none, + .none, .none, .none, .none, + }; + pub const three_integers: [8]Class = .{ + .integer, .integer, .integer, .none, + .none, .none, .none, .none, + }; + pub const four_integers: [8]Class = .{ + .integer, .integer, .integer, .integer, + .none, .none, .none, .none, + }; + pub const len_integers: [8]Class = .{ + .integer_per_element, .none, .none, .none, + .none, .none, .none, .none, + }; + + pub const @"f16" = @"f64"; + pub const @"f32": [8]Class = .{ + .float, .none, .none, .none, + .none, .none, .none, .none, + }; + pub const @"f64": [8]Class = .{ + .sse, .none, .none, .none, + .none, .none, .none, .none, + }; + pub const @"f80": [8]Class = .{ + .x87, .x87up, .none, .none, + .none, .none, .none, .none, + }; + pub const @"f128": [8]Class = .{ + .sse, .sseup, .none, .none, + .none, .none, .none, .none, + }; + + /// COMPLEX_X87: This class consists of types that will be returned via the x87 + /// FPU. + pub const complex_x87: [8]Class = .{ + .x87, .x87up, .x87, .x87up, + .none, .none, .none, .none, + }; + + pub const stack: [8]Class = .{ + .memory, .none, .none, .none, + .none, .none, .none, .none, + }; + + pub fn isX87(class: Class) bool { + return switch (class) { + .x87, .x87up => true, + else => false, + }; + } + + /// Combine a field class with the prev one. + fn combineSystemV(prev_class: Class, next_class: Class) Class { + // "If both classes are equal, this is the resulting class." + if (prev_class == next_class) + return if (prev_class == .float) .float_combine else prev_class; + + // "If one of the classes is NO_CLASS, the resulting class + // is the other class." + if (prev_class == .none) return next_class; + + // "If one of the classes is MEMORY, the result is the MEMORY class." + if (prev_class == .memory or next_class == .memory) return .memory; + + // "If one of the classes is INTEGER, the result is the INTEGER." + if (prev_class == .integer or next_class == .integer) return .integer; + + // "If one of the classes is X87, X87UP, COMPLEX_X87 class, + // MEMORY is used as class." + if (prev_class.isX87() or next_class.isX87()) return .memory; + + // "Otherwise class SSE is used." + return .sse; + } +}; + +pub fn classifyWindows(ty: Type, zcu: *Zcu, target: *const std.Target) Class { + // https://docs.microsoft.com/en-gb/cpp/build/x64-calling-convention?view=vs-2017 + // "There's a strict one-to-one correspondence between a function call's arguments + // and the registers used for those arguments. Any argument that doesn't fit in 8 + // bytes, or isn't 1, 2, 4, or 8 bytes, must be passed by reference. A single argument + // is never spread across multiple registers." + // "All floating point operations are done using the 16 XMM registers." + // "Structs and unions of size 8, 16, 32, or 64 bits, and __m64 types, are passed + // as if they were integers of the same size." + return switch (ty.zigTypeTag(zcu)) { + .pointer, + .int, + .bool, + .@"enum", + .void, + .noreturn, + .error_set, + .@"struct", + .@"union", + .optional, + .array, + .error_union, + .@"anyframe", + .frame, + => switch (ty.abiSize(zcu)) { + 0 => unreachable, + 1, 2, 4, 8 => .integer, + else => switch (ty.zigTypeTag(zcu)) { + .int => .win_i128, + .@"struct", .@"union" => if (ty.containerLayout(zcu) == .@"packed") + .win_i128 + else + .memory, + else => .memory, + }, + }, + + .float => switch (ty.floatBits(target)) { + 16, 32, 64, 128 => .sse, + 80 => .memory, + else => unreachable, + }, + .vector => .sse, + + .type, + .comptime_float, + .comptime_int, + .undefined, + .null, + .@"fn", + .@"opaque", + .enum_literal, + => unreachable, + }; +} + +pub const Context = enum { ret, arg, other }; + +/// There are a maximum of 8 possible return slots. Returned values are in +/// the beginning of the array; unused slots are filled with .none. +pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Context) [8]Class { + switch (ty.zigTypeTag(zcu)) { + .pointer => switch (ty.ptrSize(zcu)) { + .slice => return Class.two_integers, + else => return Class.one_integer, + }, + .int, .@"enum", .error_set => { + const bits = ty.intInfo(zcu).bits; + if (bits <= 64 * 1) return Class.one_integer; + if (bits <= 64 * 2) return Class.two_integers; + if (bits <= 64 * 3) return Class.three_integers; + if (bits <= 64 * 4) return Class.four_integers; + return Class.stack; + }, + .bool, .void, .noreturn => return Class.one_integer, + .float => switch (ty.floatBits(target)) { + 16 => { + if (ctx == .other) return Class.stack; + // TODO clang doesn't allow __fp16 as .ret or .arg + return Class.f16; + }, + 32 => return Class.f32, + 64 => return Class.f64, + // "Arguments of types __float128, _Decimal128 and __m128 are + // split into two halves. The least significant ones belong + // to class SSE, the most significant one to class SSEUP." + 128 => return Class.f128, + // "The 64-bit mantissa of arguments of type long double + // belongs to class X87, the 16-bit exponent plus 6 bytes + // of padding belongs to class X87UP." + 80 => return Class.f80, + else => unreachable, + }, + .vector => { + const elem_ty = ty.childType(zcu); + const bits = elem_ty.bitSize(zcu) * ty.arrayLen(zcu); + if (elem_ty.toIntern() == .bool_type) { + if (bits <= 32) return Class.one_integer; + if (bits <= 64) return Class.f64; + if (ctx == .other) return Class.stack; + if (bits <= 128) return Class.len_integers; + if (bits <= 256 and target.cpu.has(.x86, .avx)) return Class.len_integers; + if (bits <= 512 and target.cpu.has(.x86, .avx512f)) return Class.len_integers; + return Class.stack; + } + if (elem_ty.isRuntimeFloat() and elem_ty.floatBits(target) == 80) { + if (bits <= 80 * 1) return Class.f80; + if (bits <= 80 * 2) return Class.complex_x87; + return Class.stack; + } + if (bits <= 64 * 1) return .{ + .sse, .none, .none, .none, + .none, .none, .none, .none, + }; + if (bits <= 64 * 2) return .{ + .sse, .sseup, .none, .none, + .none, .none, .none, .none, + }; + if (ctx == .arg and !target.cpu.has(.x86, .avx)) return Class.stack; + if (bits <= 64 * 3) return .{ + .sse, .sseup, .sseup, .none, + .none, .none, .none, .none, + }; + if (bits <= 64 * 4) return .{ + .sse, .sseup, .sseup, .sseup, + .none, .none, .none, .none, + }; + if (ctx == .arg and !target.cpu.has(.x86, .avx512f)) return Class.stack; + if (bits <= 64 * 5) return .{ + .sse, .sseup, .sseup, .sseup, + .sseup, .none, .none, .none, + }; + if (bits <= 64 * 6) return .{ + .sse, .sseup, .sseup, .sseup, + .sseup, .sseup, .none, .none, + }; + if (bits <= 64 * 7) return .{ + .sse, .sseup, .sseup, .sseup, + .sseup, .sseup, .sseup, .none, + }; + if (bits <= 64 * 8 or (ctx == .ret and bits <= @as(u64, if (target.cpu.has(.x86, .avx512f)) + 64 * 32 + else if (target.cpu.has(.x86, .avx)) + 64 * 16 + else + 64 * 8))) return .{ + .sse, .sseup, .sseup, .sseup, + .sseup, .sseup, .sseup, .sseup, + }; + return Class.stack; + }, + .optional => { + if (ty.optionalReprIsPayload(zcu)) { + return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx); + } + return Class.stack; + }, + .@"struct", .@"union" => { + // "If the size of an object is larger than eight eightbytes, or + // it contains unaligned fields, it has class MEMORY" + // "If the size of the aggregate exceeds a single eightbyte, each is classified + // separately.". + const ty_size = ty.abiSize(zcu); + switch (ty.containerLayout(zcu)) { + .auto => unreachable, + .@"extern" => {}, + .@"packed" => { + if (ty_size <= 8) return Class.one_integer; + if (ty_size <= 16) return Class.two_integers; + unreachable; // frontend should not have allowed this type as extern + }, + } + if (ty_size > 64) return Class.stack; + + var result: [8]Class = @splat(.none); + _ = if (zcu.typeToStruct(ty)) |loaded_struct| + classifySystemVStruct(&result, 0, loaded_struct, zcu, target) + else if (zcu.typeToUnion(ty)) |loaded_union| + classifySystemVUnion(&result, 0, loaded_union, zcu, target) + else + unreachable; + + // Post-merger cleanup + + // "If one of the classes is MEMORY, the whole argument is passed in memory" + // "If X87UP is not preceded by X87, the whole argument is passed in memory." + for (result, 0..) |class, i| switch (class) { + .memory => return Class.stack, + .x87up => if (i == 0 or result[i - 1] != .x87) return Class.stack, + else => continue, + }; + // "If the size of the aggregate exceeds two eightbytes and the first eight- + // byte isn’t SSE or any other eightbyte isn’t SSEUP, the whole argument + // is passed in memory." + if (ty_size > 16 and (result[0] != .sse or + std.mem.indexOfNone(Class, result[1..], &.{ .sseup, .none }) != null)) return Class.stack; + + // "If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE." + for (&result, 0..) |*item, i| { + if (item.* == .sseup) switch (result[i - 1]) { + .sse, .sseup => continue, + else => item.* = .sse, + }; + } + return result; + }, + .array => { + const ty_size = ty.abiSize(zcu); + if (ty_size <= 8) return Class.one_integer; + if (ty_size <= 16) return Class.two_integers; + return Class.stack; + }, + else => unreachable, + } +} + +fn classifySystemVStruct( + result: *[8]Class, + starting_byte_offset: u64, + loaded_struct: InternPool.LoadedStructType, + zcu: *Zcu, + target: *const std.Target, +) u64 { + const ip = &zcu.intern_pool; + var byte_offset = starting_byte_offset; + var field_it = loaded_struct.iterateRuntimeOrder(ip); + while (field_it.next()) |field_index| { + const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_align = loaded_struct.fieldAlign(ip, field_index); + byte_offset = std.mem.alignForward( + u64, + byte_offset, + field_align.toByteUnits() orelse field_ty.abiAlignment(zcu).toByteUnits().?, + ); + if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { + switch (field_loaded_struct.layout) { + .auto => unreachable, + .@"extern" => { + byte_offset = classifySystemVStruct(result, byte_offset, field_loaded_struct, zcu, target); + continue; + }, + .@"packed" => {}, + } + } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| { + switch (field_loaded_union.flagsUnordered(ip).layout) { + .auto => unreachable, + .@"extern" => { + byte_offset = classifySystemVUnion(result, byte_offset, field_loaded_union, zcu, target); + continue; + }, + .@"packed" => {}, + } + } + const field_classes = std.mem.sliceTo(&classifySystemV(field_ty, zcu, target, .other), .none); + for (result[@intCast(byte_offset / 8)..][0..field_classes.len], field_classes) |*result_class, field_class| + result_class.* = result_class.combineSystemV(field_class); + byte_offset += field_ty.abiSize(zcu); + } + const final_byte_offset = starting_byte_offset + loaded_struct.sizeUnordered(ip); + std.debug.assert(final_byte_offset == std.mem.alignForward( + u64, + byte_offset, + loaded_struct.flagsUnordered(ip).alignment.toByteUnits().?, + )); + return final_byte_offset; +} + +fn classifySystemVUnion( + result: *[8]Class, + starting_byte_offset: u64, + loaded_union: InternPool.LoadedUnionType, + zcu: *Zcu, + target: *const std.Target, +) u64 { + const ip = &zcu.intern_pool; + for (0..loaded_union.field_types.len) |field_index| { + const field_ty = Type.fromInterned(loaded_union.field_types.get(ip)[field_index]); + if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { + switch (field_loaded_struct.layout) { + .auto => unreachable, + .@"extern" => { + _ = classifySystemVStruct(result, starting_byte_offset, field_loaded_struct, zcu, target); + continue; + }, + .@"packed" => {}, + } + } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| { + switch (field_loaded_union.flagsUnordered(ip).layout) { + .auto => unreachable, + .@"extern" => { + _ = classifySystemVUnion(result, starting_byte_offset, field_loaded_union, zcu, target); + continue; + }, + .@"packed" => {}, + } + } + const field_classes = std.mem.sliceTo(&classifySystemV(field_ty, zcu, target, .other), .none); + for (result[@intCast(starting_byte_offset / 8)..][0..field_classes.len], field_classes) |*result_class, field_class| + result_class.* = result_class.combineSystemV(field_class); + } + return starting_byte_offset + loaded_union.sizeUnordered(ip); +} + +pub const zigcc = struct { + pub const stack_align: ?InternPool.Alignment = null; + pub const return_in_regs = true; + pub const params_in_regs = true; + + const volatile_gpr = gp_regs.len - 5; + const volatile_x87 = x87_regs.len - 1; + const volatile_sse = sse_avx_regs.len; + + /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them + /// for anything else but stack offset tracking therefore we exclude them from this set. + pub const callee_preserved_regs = gp_regs[volatile_gpr..] ++ x87_regs[volatile_x87 .. x87_regs.len - 1] ++ sse_avx_regs[volatile_sse..]; + /// These registers need to be preserved (saved on the stack) and restored by the caller before + /// the caller relinquishes control to a subroutine via call instruction (or similar). + /// In other words, these registers are free to use by the callee. + pub const caller_preserved_regs = gp_regs[0..volatile_gpr] ++ x87_regs[0..volatile_x87] ++ sse_avx_regs[0..volatile_sse]; + + const int_param_regs = gp_regs[0 .. volatile_gpr - 1]; + const x87_param_regs = x87_regs[0..volatile_x87]; + const sse_param_regs = sse_avx_regs[0 .. volatile_sse / 2]; + const int_return_regs = gp_regs[0..volatile_gpr]; + const x87_return_regs = x87_regs[0..volatile_x87]; + const sse_return_regs = sse_avx_regs[0..volatile_gpr]; +}; + +pub const SysV = struct { + /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them + /// for anything else but stack offset tracking therefore we exclude them from this set. + pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 }; + /// These registers need to be preserved (saved on the stack) and restored by the caller before + /// the caller relinquishes control to a subroutine via call instruction (or similar). + /// In other words, these registers are free to use by the callee. + pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs; + + pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; + pub const c_abi_x87_param_regs = x87_regs[0..0]; + pub const c_abi_sse_param_regs = sse_avx_regs[0..8]; + pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; + pub const c_abi_x87_return_regs = x87_regs[0..2]; + pub const c_abi_sse_return_regs = sse_avx_regs[0..4]; +}; + +pub const Win64 = struct { + /// Note that .rsp and .rbp also belong to this set, however, we never expect to use them + /// for anything else but stack offset tracking therefore we exclude them from this set. + pub const callee_preserved_regs = [_]Register{ .rbx, .rsi, .rdi, .r12, .r13, .r14, .r15 }; + /// These registers need to be preserved (saved on the stack) and restored by the caller before + /// the caller relinquishes control to a subroutine via call instruction (or similar). + /// In other words, these registers are free to use by the callee. + pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs; + + pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 }; + pub const c_abi_x87_param_regs = x87_regs[0..0]; + pub const c_abi_sse_param_regs = sse_avx_regs[0..4]; + pub const c_abi_int_return_regs = [_]Register{.rax}; + pub const c_abi_x87_return_regs = x87_regs[0..0]; + pub const c_abi_sse_return_regs = sse_avx_regs[0..1]; +}; + +pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.callee_preserved_regs, + .x86_64_sysv => &SysV.callee_preserved_regs, + .x86_64_win => &Win64.callee_preserved_regs, + else => unreachable, + }; +} + +pub fn getCallerPreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.caller_preserved_regs, + .x86_64_sysv => &SysV.caller_preserved_regs, + .x86_64_win => &Win64.caller_preserved_regs, + else => unreachable, + }; +} + +pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.int_param_regs, + .x86_64_sysv => &SysV.c_abi_int_param_regs, + .x86_64_win => &Win64.c_abi_int_param_regs, + else => unreachable, + }; +} + +pub fn getCAbiX87ParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.x87_param_regs, + .x86_64_sysv => SysV.c_abi_x87_param_regs, + .x86_64_win => Win64.c_abi_x87_param_regs, + else => unreachable, + }; +} + +pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention.Tag, target: *const std.Target) []const Register { + return switch (cc) { + .auto => switch (target.cpu.arch) { + else => unreachable, + .x86 => zigcc.sse_param_regs[0 .. zigcc.sse_param_regs.len / 2], + .x86_64 => zigcc.sse_param_regs, + }, + .x86_64_sysv => SysV.c_abi_sse_param_regs, + .x86_64_win => Win64.c_abi_sse_param_regs, + else => unreachable, + }; +} + +pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.int_return_regs, + .x86_64_sysv => &SysV.c_abi_int_return_regs, + .x86_64_win => &Win64.c_abi_int_return_regs, + else => unreachable, + }; +} + +pub fn getCAbiX87ReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.x87_return_regs, + .x86_64_sysv => SysV.c_abi_x87_return_regs, + .x86_64_win => Win64.c_abi_x87_return_regs, + else => unreachable, + }; +} + +pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register { + return switch (cc) { + .auto => zigcc.sse_return_regs, + .x86_64_sysv => SysV.c_abi_sse_return_regs, + .x86_64_win => Win64.c_abi_sse_return_regs, + else => unreachable, + }; +} + +pub fn getCAbiLinkerScratchReg(cc: std.builtin.CallingConvention.Tag) Register { + return switch (cc) { + .auto => zigcc.int_return_regs[zigcc.int_return_regs.len - 1], + .x86_64_sysv => SysV.c_abi_int_return_regs[0], + .x86_64_win => Win64.c_abi_int_return_regs[0], + else => unreachable, + }; +} + +const gp_regs = [_]Register{ + .rax, .rdx, .rbx, .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15, +}; +const x87_regs = [_]Register{ + .st0, .st1, .st2, .st3, .st4, .st5, .st6, .st7, +}; +const sse_avx_regs = [_]Register{ + .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, + .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, +}; +const allocatable_regs = gp_regs ++ x87_regs[0 .. x87_regs.len - 1] ++ sse_avx_regs; +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, allocatable_regs); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.general_purpose)) set.set(index); + break :blk set; + }; + pub const gphi: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.gphi)) set.set(index); + break :blk set; + }; + pub const x87: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.x87)) set.set(index); + break :blk set; + }; + pub const sse: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + for (allocatable_regs, 0..) |reg, index| if (reg.isClass(.sse)) set.set(index); + break :blk set; + }; +}; + +const builtin = @import("builtin"); +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; + +const InternPool = @import("../../InternPool.zig"); +const Register = @import("bits.zig").Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; +const Type = @import("../../Type.zig"); +const Value = @import("../../Value.zig"); +const Zcu = @import("../../Zcu.zig"); diff --git a/src/codegen/x86_64/bits.zig b/src/codegen/x86_64/bits.zig new file mode 100644 index 0000000000000000000000000000000000000000..05048109335e0256d91c5f350d7fa900ce08e402 --- /dev/null +++ b/src/codegen/x86_64/bits.zig @@ -0,0 +1,912 @@ +const std = @import("std"); +const assert = std.debug.assert; +const expect = std.testing.expect; + +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; +const InternPool = @import("../../InternPool.zig"); +const link = @import("../../link.zig"); +const Mir = @import("Mir.zig"); + +/// EFLAGS condition codes +pub const Condition = enum(u5) { + /// above + a, + /// above or equal + ae, + /// below + b, + /// below or equal + be, + /// carry + c, + /// equal + e, + /// greater + g, + /// greater or equal + ge, + /// less + l, + /// less or equal + le, + /// not above + na, + /// not above or equal + nae, + /// not below + nb, + /// not below or equal + nbe, + /// not carry + nc, + /// not equal + ne, + /// not greater + ng, + /// not greater or equal + nge, + /// not less + nl, + /// not less or equal + nle, + /// not overflow + no, + /// not parity + np, + /// not sign + ns, + /// not zero + nz, + /// overflow + o, + /// parity + p, + /// parity even + pe, + /// parity odd + po, + /// sign + s, + /// zero + z, + + // Pseudo conditions + /// zero and not parity + z_and_np, + /// not zero or parity + nz_or_p, + + /// Converts a std.math.CompareOperator into a condition flag, + /// i.e. returns the condition that is true iff the result of the + /// comparison is true. Assumes signed comparison + pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) Condition { + return switch (op) { + .gte => .ge, + .gt => .g, + .neq => .ne, + .lt => .l, + .lte => .le, + .eq => .e, + }; + } + + /// Converts a std.math.CompareOperator into a condition flag, + /// i.e. returns the condition that is true iff the result of the + /// comparison is true. Assumes unsigned comparison + pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) Condition { + return switch (op) { + .gte => .ae, + .gt => .a, + .neq => .ne, + .lt => .b, + .lte => .be, + .eq => .e, + }; + } + + pub fn fromCompareOperator( + signedness: std.builtin.Signedness, + op: std.math.CompareOperator, + ) Condition { + return switch (signedness) { + .signed => fromCompareOperatorSigned(op), + .unsigned => fromCompareOperatorUnsigned(op), + }; + } + + /// Returns the condition which is true iff the given condition is false + pub fn negate(cond: Condition) Condition { + return switch (cond) { + .a => .na, + .ae => .nae, + .b => .nb, + .be => .nbe, + .c => .nc, + .e => .ne, + .g => .ng, + .ge => .nge, + .l => .nl, + .le => .nle, + .na => .a, + .nae => .ae, + .nb => .b, + .nbe => .be, + .nc => .c, + .ne => .e, + .ng => .g, + .nge => .ge, + .nl => .l, + .nle => .le, + .no => .o, + .np => .p, + .ns => .s, + .nz => .z, + .o => .no, + .p => .np, + .pe => .po, + .po => .pe, + .s => .ns, + .z => .nz, + + .z_and_np => .nz_or_p, + .nz_or_p => .z_and_np, + }; + } + + /// Returns the equivalent condition when the operands are swapped. + pub fn commute(cond: Condition) Condition { + return switch (cond) { + else => cond, + .a => .b, + .ae => .be, + .b => .a, + .be => .ae, + .g => .l, + .ge => .le, + .l => .g, + .le => .ge, + .na => .nb, + .nae => .nbe, + .nb => .na, + .nbe => .nae, + .ng => .nl, + .nge => .nle, + .nl => .ng, + .nle => .nge, + }; + } +}; + +/// The immediate operand of vcvtps2ph. +pub const RoundMode = packed struct(u5) { + direction: Direction = .mxcsr, + precision: enum(u1) { + normal = 0b0, + inexact = 0b1, + } = .normal, + + pub const Direction = enum(u4) { + /// Round to nearest (even) + nearest = 0b0_00, + /// Round down (toward -∞) + down = 0b0_01, + /// Round up (toward +∞) + up = 0b0_10, + /// Round toward zero (truncate) + zero = 0b0_11, + /// Use current rounding mode of MXCSR.RC + mxcsr = 0b1_00, + }; + + pub fn imm(mode: RoundMode) Immediate { + return .u(@as(@typeInfo(RoundMode).@"struct".backing_integer.?, @bitCast(mode))); + } +}; + +/// The immediate operand of cmppd, cmpps, cmpsd, and cmpss. +pub const SseFloatPredicate = enum(u3) { + /// Equal (ordered, non-signaling) + eq, + /// Less-than (ordered, signaling) + lt, + /// Less-than-or-equal (ordered, signaling) + le, + /// Unordered (non-signaling) + unord, + /// Not-equal (unordered, non-signaling) + neq, + /// Not-less-than (unordered, signaling) + nlt, + /// Not-less-than-or-equal (unordered, signaling) + nle, + /// Ordered (non-signaling) + ord, + + /// Equal (ordered, non-signaling) + pub const eq_oq: SseFloatPredicate = .eq; + /// Less-than (ordered, signaling) + pub const lt_os: SseFloatPredicate = .lt; + /// Less-than-or-equal (ordered, signaling) + pub const le_os: SseFloatPredicate = .le; + /// Unordered (non-signaling) + pub const unord_q: SseFloatPredicate = .unord; + /// Not-equal (unordered, non-signaling) + pub const neq_uq: SseFloatPredicate = .neq; + /// Not-less-than (unordered, signaling) + pub const nlt_us: SseFloatPredicate = .nlt; + /// Not-less-than-or-equal (unordered, signaling) + pub const nle_us: SseFloatPredicate = .nle; + /// Ordered (non-signaling) + pub const ord_q: SseFloatPredicate = .ord; + + pub fn imm(pred: SseFloatPredicate) Immediate { + return .u(@intFromEnum(pred)); + } +}; + +/// The immediate operand of vcmppd, vcmpps, vcmpsd, and vcmpss. +pub const VexFloatPredicate = enum(u5) { + /// Equal (ordered, non-signaling) + eq_oq, + /// Less-than (ordered, signaling) + lt_os, + /// Less-than-or-equal (ordered, signaling) + le_os, + /// Unordered (non-signaling) + unord_q, + /// Not-equal (unordered, non-signaling) + neq_uq, + /// Not-less-than (unordered, signaling) + nlt_us, + /// Not-less-than-or-equal (unordered, signaling) + nle_us, + /// Ordered (non-signaling) + ord_q, + /// Equal (unordered, non-signaling) + eq_uq, + /// Not-greater-than-or-equal (unordered, signaling) + nge_us, + /// Not-greater-than (unordered, signaling) + ngt_us, + /// False (ordered, non-signaling) + false_oq, + /// Not-equal (ordered, non-signaling) + neq_oq, + /// Greater-than-or-equal (ordered, signaling) + ge_os, + /// Greater-than (ordered, signaling) + gt_os, + /// True (unordered, non-signaling) + true_uq, + /// Equal (unordered, non-signaling) + eq_os, + /// Less-than (ordered, non-signaling) + lt_oq, + /// Less-than-or-equal (ordered, non-signaling) + le_oq, + /// Unordered (signaling) + unord_s, + /// Not-equal (unordered, signaling) + neq_us, + /// Not-less-than (unordered, non-signaling) + nlt_uq, + /// Not-less-than-or-equal (unordered, non-signaling) + nle_uq, + /// Ordered (signaling) + ord_s, + /// Equal (unordered, signaling) + eq_us, + /// Not-greater-than-or-equal (unordered, non-signaling) + nge_uq, + /// Not-greater-than (unordered, non-signaling) + ngt_uq, + /// False (ordered, signaling) + false_os, + /// Not-equal (ordered, signaling) + neq_os, + /// Greater-than-or-equal (ordered, non-signaling) + ge_oq, + /// Greater-than (ordered, non-signaling) + gt_oq, + /// True (unordered, signaling) + true_us, + + /// Equal (ordered, non-signaling) + pub const eq: VexFloatPredicate = .eq_oq; + /// Less-than (ordered, signaling) + pub const lt: VexFloatPredicate = .lt_os; + /// Less-than-or-equal (ordered, signaling) + pub const le: VexFloatPredicate = .le_os; + /// Unordered (non-signaling) + pub const unord: VexFloatPredicate = .unord_q; + /// Not-equal (unordered, non-signaling) + pub const neq: VexFloatPredicate = .neq_uq; + /// Not-less-than (unordered, signaling) + pub const nlt: VexFloatPredicate = .nlt_us; + /// Not-less-than-or-equal (unordered, signaling) + pub const nle: VexFloatPredicate = .nle_us; + /// Ordered (non-signaling) + pub const ord: VexFloatPredicate = .ord_q; + /// Not-greater-than-or-equal (unordered, signaling) + pub const nge: VexFloatPredicate = .nge_us; + /// Not-greater-than (unordered, signaling) + pub const ngt: VexFloatPredicate = .ngt_us; + /// False (ordered, non-signaling) + pub const @"false": VexFloatPredicate = .false_oq; + /// Greater-than-or-equal (ordered, signaling) + pub const ge: VexFloatPredicate = .ge_os; + /// Greater-than (ordered, signaling) + pub const gt: VexFloatPredicate = .gt_os; + /// True (unordered, non-signaling) + pub const @"true": VexFloatPredicate = .true_uq; + + pub fn imm(pred: VexFloatPredicate) Immediate { + return .u(@intFromEnum(pred)); + } +}; + +pub const Register = enum(u8) { + // zig fmt: off + rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi, + r8, r9, r10, r11, r12, r13, r14, r15, + + eax, ecx, edx, ebx, esp, ebp, esi, edi, + r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d, + + ax, cx, dx, bx, sp, bp, si, di, + r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w, + + al, cl, dl, bl, spl, bpl, sil, dil, + r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b, + + ah, ch, dh, bh, + + zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, + zmm8, zmm9, zmm10, zmm11, zmm12, zmm13, zmm14, zmm15, + zmm16, zmm17,zmm18, zmm19, zmm20, zmm21, zmm22, zmm23, + zmm24, zmm25,zmm26, zmm27, zmm28, zmm29, zmm30, zmm31, + + ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, + ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15, + ymm16, ymm17,ymm18, ymm19, ymm20, ymm21, ymm22, ymm23, + ymm24, ymm25,ymm26, ymm27, ymm28, ymm29, ymm30, ymm31, + + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + xmm16, xmm17,xmm18, xmm19, xmm20, xmm21, xmm22, xmm23, + xmm24, xmm25,xmm26, xmm27, xmm28, xmm29, xmm30, xmm31, + + mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, + + st0, st1, st2, st3, st4, st5, st6, st7, + + es, cs, ss, ds, fs, gs, + + rip, eip, ip, + + cr0, cr1, cr2, cr3, cr4, cr5, cr6, cr7, + cr8, cr9, cr10, cr11, cr12, cr13, cr14, cr15, + + dr0, dr1, dr2, dr3, dr4, dr5, dr6, dr7, + dr8, dr9, dr10, dr11, dr12, dr13, dr14, dr15, + + none, + // zig fmt: on + + pub const Class = enum { + general_purpose, + gphi, + segment, + x87, + mmx, + sse, + ip, + cr, + dr, + }; + + pub fn class(reg: Register) Class { + return switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => .general_purpose, + @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => .general_purpose, + @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => .general_purpose, + @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => .general_purpose, + @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => .gphi, + + @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm31) => .sse, + @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm31) => .sse, + @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm31) => .sse, + @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => .mmx, + @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => .x87, + + @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => .segment, + @intFromEnum(Register.rip) ... @intFromEnum(Register.ip) => .ip, + @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => .cr, + @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => .dr, + + else => unreachable, + // zig fmt: on + }; + } + + pub inline fn isClass(reg: Register, rc: Class) bool { + switch (rc) { + else => return reg.class() == rc, + .gphi => { + const reg_id = reg.id(); + return (reg_id >= comptime Register.ah.id()) and reg_id <= comptime Register.bh.id(); + }, + } + } + + pub fn id(reg: Register) u7 { + const base = switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => @intFromEnum(Register.rax), + @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), + @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), + @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), + @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah), + + @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm31) => @intFromEnum(Register.zmm0) - 16, + @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm31) => @intFromEnum(Register.ymm0) - 16, + @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm31) => @intFromEnum(Register.xmm0) - 16, + @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => @intFromEnum(Register.mm0) - 48, + @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => @intFromEnum(Register.st0) - 56, + @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => @intFromEnum(Register.es) - 64, + @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => @intFromEnum(Register.cr0) - 70, + @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => @intFromEnum(Register.dr0) - 86, + + else => unreachable, + // zig fmt: on + }; + return @intCast(@intFromEnum(reg) - base); + } + + pub fn size(reg: Register) Memory.Size { + return switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => .qword, + @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => .dword, + @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => .word, + @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => .byte, + @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => .byte, + + @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm15) => .zword, + @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => .yword, + @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => .xword, + @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => .qword, + @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => .tbyte, + + @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => .word, + + @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => .gpr, + @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => .gpr, + + else => unreachable, + // zig fmt: on + }; + } + + pub fn isExtended(reg: Register) bool { + return switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.r8) ... @intFromEnum(Register.r15) => true, + @intFromEnum(Register.r8d) ... @intFromEnum(Register.r15d) => true, + @intFromEnum(Register.r8w) ... @intFromEnum(Register.r15w) => true, + @intFromEnum(Register.r8b) ... @intFromEnum(Register.r15b) => true, + + @intFromEnum(Register.zmm8) ... @intFromEnum(Register.zmm31) => true, + @intFromEnum(Register.ymm8) ... @intFromEnum(Register.ymm31) => true, + @intFromEnum(Register.xmm8) ... @intFromEnum(Register.xmm31) => true, + + @intFromEnum(Register.cr8) ... @intFromEnum(Register.cr15) => true, + @intFromEnum(Register.dr8) ... @intFromEnum(Register.dr15) => true, + + else => false, + // zig fmt: on + }; + } + + pub fn enc(reg: Register) u5 { + const base = switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => @intFromEnum(Register.rax), + @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), + @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), + @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), + @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah) - 4, + + @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0), + @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0), + @intFromEnum(Register.mm0) ... @intFromEnum(Register.mm7) => @intFromEnum(Register.mm0), + @intFromEnum(Register.st0) ... @intFromEnum(Register.st7) => @intFromEnum(Register.st0), + + @intFromEnum(Register.es) ... @intFromEnum(Register.gs) => @intFromEnum(Register.es), + + @intFromEnum(Register.cr0) ... @intFromEnum(Register.cr15) => @intFromEnum(Register.cr0), + @intFromEnum(Register.dr0) ... @intFromEnum(Register.dr15) => @intFromEnum(Register.dr0), + + else => unreachable, + // zig fmt: on + }; + return @truncate(@intFromEnum(reg) - base); + } + + pub fn toBitSize(reg: Register, bit_size: u64) Register { + return switch (bit_size) { + 8 => reg.to8(), + 16 => reg.to16(), + 32 => reg.to32(), + 64 => reg.to64(), + 80 => reg.to80(), + 128 => reg.to128(), + 256 => reg.to256(), + 512 => reg.to512(), + else => unreachable, + }; + } + + pub fn toSize(reg: Register, new_size: Memory.Size, target: *const std.Target) Register { + return switch (new_size) { + .none => unreachable, + .ptr => reg.toBitSize(target.ptrBitWidth()), + .gpr => switch (target.cpu.arch) { + else => unreachable, + .x86 => reg.to32(), + .x86_64 => reg.to64(), + }, + .low_byte => reg.toLo8(), + .high_byte => reg.toHi8(), + .byte => reg.to8(), + .word => reg.to16(), + .dword => reg.to32(), + .qword => reg.to64(), + .tbyte => reg.to80(), + .xword => reg.to128(), + .yword => reg.to256(), + .zword => reg.to512(), + }; + } + + fn gpBase(reg: Register) u7 { + return switch (@intFromEnum(reg)) { + // zig fmt: off + @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => @intFromEnum(Register.rax), + @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), + @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), + @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), + @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah), + else => unreachable, + // zig fmt: on + }; + } + + pub fn to64(reg: Register) Register { + return switch (reg.class()) { + .general_purpose, .gphi => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.rax)), + .segment => unreachable, + .x87, .mmx, .cr, .dr => reg, + .sse => reg.to128(), + .ip => .rip, + }; + } + + pub fn to32(reg: Register) Register { + return switch (reg.class()) { + .general_purpose, .gphi => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.eax)), + .segment => unreachable, + .x87, .mmx, .cr, .dr => reg, + .sse => reg.to128(), + .ip => .eip, + }; + } + + pub fn to16(reg: Register) Register { + return switch (reg.class()) { + .general_purpose, .gphi => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.ax)), + .segment, .x87, .mmx, .cr, .dr => reg, + .sse => reg.to128(), + .ip => .ip, + }; + } + + pub fn to8(reg: Register) Register { + return switch (reg.class()) { + .general_purpose => reg.toLo8(), + .gphi, .segment, .x87, .mmx, .cr, .dr => reg, + .sse => reg.to128(), + .ip => .ip, + }; + } + + pub fn toLo8(reg: Register) Register { + return @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.al)); + } + + pub fn toHi8(reg: Register) Register { + assert(reg.isClass(.gphi)); + return @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.ah)); + } + + pub fn to80(reg: Register) Register { + assert(reg.isClass(.x87)); + return reg; + } + + fn sseBase(reg: Register) u8 { + assert(reg.isClass(.sse)); + return switch (@intFromEnum(reg)) { + @intFromEnum(Register.zmm0)...@intFromEnum(Register.zmm31) => @intFromEnum(Register.zmm0), + @intFromEnum(Register.ymm0)...@intFromEnum(Register.ymm31) => @intFromEnum(Register.ymm0), + @intFromEnum(Register.xmm0)...@intFromEnum(Register.xmm31) => @intFromEnum(Register.xmm0), + else => unreachable, + }; + } + + pub fn to512(reg: Register) Register { + return @enumFromInt(@intFromEnum(reg) - reg.sseBase() + @intFromEnum(Register.zmm0)); + } + + pub fn to256(reg: Register) Register { + return @enumFromInt(@intFromEnum(reg) - reg.sseBase() + @intFromEnum(Register.ymm0)); + } + + pub fn to128(reg: Register) Register { + return @enumFromInt(@intFromEnum(reg) - reg.sseBase() + @intFromEnum(Register.xmm0)); + } + + /// DWARF register encoding + pub fn dwarfNum(reg: Register) u6 { + return switch (reg.class()) { + .general_purpose, .gphi => if (reg.isExtended()) + reg.enc() + else + @as(u3, @truncate(@as(u24, 0o54673120) >> @as(u5, reg.enc()) * 3)), + .sse => 17 + @as(u6, reg.enc()), + .x87 => 33 + @as(u6, reg.enc()), + .mmx => 41 + @as(u6, reg.enc()), + .segment => 50 + @as(u6, reg.enc()), + .ip => 16, + .cr, .dr => unreachable, + }; + } +}; + +test "Register id - different classes" { + try expect(Register.al.id() == Register.ax.id()); + try expect(Register.ah.id() != Register.spl.id()); + try expect(Register.ax.id() == Register.eax.id()); + try expect(Register.eax.id() == Register.rax.id()); + + try expect(Register.ymm0.id() == 0b10000); + try expect(Register.ymm0.id() != Register.rax.id()); + try expect(Register.xmm0.id() == Register.ymm0.id()); + try expect(Register.xmm0.id() != Register.mm0.id()); + try expect(Register.mm0.id() != Register.st0.id()); +} + +test "Register enc - different classes" { + try expect(Register.al.enc() == Register.ax.enc()); + try expect(Register.ah.enc() == Register.spl.enc()); + try expect(Register.ax.enc() == Register.eax.enc()); + try expect(Register.eax.enc() == Register.rax.enc()); + try expect(Register.ymm0.enc() == Register.rax.enc()); + try expect(Register.xmm0.enc() == Register.ymm0.enc()); + try expect(Register.es.enc() == Register.rax.enc()); +} + +test "Register classes" { + try expect(Register.r11.isClass(.general_purpose)); + try expect(Register.rdx.isClass(.gphi)); + try expect(!Register.dil.isClass(.gphi)); + try expect(Register.ymm11.isClass(.sse)); + try expect(Register.mm3.isClass(.mmx)); + try expect(Register.st3.isClass(.x87)); + try expect(Register.fs.isClass(.segment)); +} + +pub const FrameIndex = enum(u32) { + // This index refers to the start of the arguments passed to this function + args_frame, + // This index refers to the return address pushed by a `call` and popped by a `ret`. + ret_addr, + // This index refers to the base pointer pushed in the prologue and popped in the epilogue. + base_ptr, + // This index refers to the entire stack frame. + stack_frame, + // This index refers to the start of the call frame for arguments passed to called functions + call_frame, + // Other indices are used for local variable stack slots + _, + + pub const named_count = @typeInfo(FrameIndex).@"enum".fields.len; + + pub fn isNamed(fi: FrameIndex) bool { + return @intFromEnum(fi) < named_count; + } + + pub fn format(fi: FrameIndex, writer: *std.Io.Writer) std.Io.Writer.Error!void { + if (fi.isNamed()) { + try writer.print("FrameIndex.{t}", .{fi}); + } else { + try writer.print("FrameIndex({d})", .{@intFromEnum(fi)}); + } + } +}; + +pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 }; + +pub const RegisterOffset = struct { reg: Register, off: i32 = 0 }; + +pub const NavOffset = struct { index: InternPool.Nav.Index, off: i32 = 0 }; + +pub const Memory = struct { + base: Base = .none, + mod: Mod = .{ .rm = .{} }, + + pub const Base = union(enum(u4)) { + none, + reg: Register, + frame: FrameIndex, + table, + rip_inst: Mir.Inst.Index, + nav: InternPool.Nav.Index, + uav: InternPool.Key.Ptr.BaseAddr.Uav, + lazy_sym: link.File.LazySymbol, + extern_func: Mir.NullTerminatedString, + + pub const Tag = @typeInfo(Base).@"union".tag_type.?; + }; + + pub const Mod = union(enum(u1)) { + rm: Rm, + off: u64, + + pub const Rm = struct { + size: Size = .none, + index: Register = .none, + scale: Scale = .@"1", + disp: i32 = 0, + }; + }; + + pub const Size = enum(u4) { + none, + ptr, + gpr, + low_byte, + high_byte, + byte, + word, + dword, + qword, + tbyte, + xword, + yword, + zword, + + pub fn fromSize(size: u32) Size { + return switch (size) { + 1...1 => .byte, + 2...2 => .word, + 3...4 => .dword, + 5...8 => .qword, + 9...16 => .xword, + 17...32 => .yword, + 33...64 => .zword, + else => unreachable, + }; + } + + pub fn fromBitSize(bit_size: u64) Size { + return switch (bit_size) { + 8 => .byte, + 16 => .word, + 32 => .dword, + 64 => .qword, + 80 => .tbyte, + 128 => .xword, + 256 => .yword, + 512 => .zword, + else => unreachable, + }; + } + + pub fn bitSize(s: Size, target: *const std.Target) u64 { + return switch (s) { + .none => 0, + .ptr => target.ptrBitWidth(), + .gpr => switch (target.cpu.arch) { + else => unreachable, + .x86 => 32, + .x86_64 => 64, + }, + .low_byte, .high_byte, .byte => 8, + .word => 16, + .dword => 32, + .qword => 64, + .tbyte => 80, + .xword => 128, + .yword => 256, + .zword => 512, + }; + } + + pub fn format(s: Size, writer: *std.Io.Writer) std.Io.Writer.Error!void { + if (s == .none) return; + try writer.writeAll(@tagName(s)); + switch (s) { + .none => unreachable, + .ptr, .gpr => {}, + else => { + try writer.writeByte(' '); + try writer.writeAll("ptr"); + }, + } + } + }; + + pub const Scale = enum(u2) { + @"1", + @"2", + @"4", + @"8", + + pub fn fromFactor(factor: u4) Scale { + return switch (factor) { + else => unreachable, + 1 => .@"1", + 2 => .@"2", + 4 => .@"4", + 8 => .@"8", + }; + } + + pub fn toFactor(scale: Scale) u4 { + return switch (scale) { + .@"1" => 1, + .@"2" => 2, + .@"4" => 4, + .@"8" => 8, + }; + } + + pub fn fromLog2(log2: u2) Scale { + return @enumFromInt(log2); + } + + pub fn toLog2(scale: Scale) u2 { + return @intFromEnum(scale); + } + }; +}; + +pub const Immediate = union(enum) { + signed: i32, + unsigned: u64, + nav: NavOffset, + uav: InternPool.Key.Ptr.BaseAddr.Uav, + lazy_sym: link.File.LazySymbol, + extern_func: Mir.NullTerminatedString, + + pub fn u(x: u64) Immediate { + return .{ .unsigned = x }; + } + + pub fn s(x: i32) Immediate { + return .{ .signed = x }; + } + + pub fn format(imm: Immediate, writer: *std.Io.Writer) std.Io.Writer.Error!void { + switch (imm) { + inline else => |int| try writer.print("{d}", .{int}), + .nav => |nav_off| try writer.print("Nav({d}) + {d}", .{ @intFromEnum(nav_off.nav), nav_off.off }), + .uav => |uav| try writer.print("Uav({d})", .{@intFromEnum(uav.val)}), + .lazy_sym => |lazy_sym| try writer.print("LazySym({s}, {d})", .{ @tagName(lazy_sym.kind), @intFromEnum(lazy_sym.ty) }), + .extern_func => |extern_func| try writer.print("ExternFunc({d})", .{@intFromEnum(extern_func)}), + } + } +}; diff --git a/src/codegen/x86_64/encoder.zig b/src/codegen/x86_64/encoder.zig new file mode 100644 index 0000000000000000000000000000000000000000..1d9c67175ef204933a5463393cad189d0e59e7fa --- /dev/null +++ b/src/codegen/x86_64/encoder.zig @@ -0,0 +1,2789 @@ +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.x86_64_encoder); +const math = std.math; +const testing = std.testing; +const Writer = std.Io.Writer; + +const bits = @import("bits.zig"); +const Encoding = @import("Encoding.zig"); +const FrameIndex = bits.FrameIndex; +const Register = bits.Register; +const Symbol = bits.Symbol; + +pub const Instruction = struct { + prefix: Prefix = .none, + encoding: Encoding, + ops: [4]Operand = .{.none} ** 4, + + pub const Mnemonic = Encoding.Mnemonic; + + pub const Prefix = enum(u3) { + none, + lock, + rep, + repe, + repz, + repne, + repnz, + directive, + }; + + pub const Immediate = union(enum) { + signed: i32, + unsigned: u64, + + pub fn u(x: u64) Immediate { + return .{ .unsigned = x }; + } + + pub fn s(x: i32) Immediate { + return .{ .signed = x }; + } + + pub fn asSigned(imm: Immediate, bit_size: u64) i64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(i8, @intCast(x)), + 16 => @as(i16, @intCast(x)), + 32, 64 => x, + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), + 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), + 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), + 64 => @bitCast(x), + else => unreachable, + }, + }; + } + + pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))), + 16 => @as(u16, @bitCast(@as(i16, @intCast(x)))), + 32, 64 => @as(u32, @bitCast(x)), + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(u8, @intCast(x)), + 16 => @as(u16, @intCast(x)), + 32 => @as(u32, @intCast(x)), + 64 => x, + else => unreachable, + }, + }; + } + }; + + pub const Memory = union(enum) { + sib: Sib, + rip: Rip, + moffs: Moffs, + + pub const Base = bits.Memory.Base; + + pub const ScaleIndex = struct { + scale: u4, + index: Register, + + const none = ScaleIndex{ .scale = 0, .index = undefined }; + }; + + pub const PtrSize = bits.Memory.Size; + + pub const Sib = struct { + ptr_size: PtrSize, + base: Base, + scale_index: ScaleIndex, + disp: i32, + }; + + pub const Rip = struct { + ptr_size: PtrSize, + disp: i32, + }; + + pub const Moffs = struct { + seg: Register, + offset: u64, + }; + + pub fn initMoffs(reg: Register, offset: u64) Memory { + assert(reg.isClass(.segment)); + return .{ .moffs = .{ .seg = reg, .offset = offset } }; + } + + pub fn initSib(ptr_size: PtrSize, args: struct { + disp: i32 = 0, + base: Base = .none, + scale_index: ?ScaleIndex = null, + }) Memory { + if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale)); + return .{ .sib = .{ + .base = args.base, + .disp = args.disp, + .ptr_size = ptr_size, + .scale_index = if (args.scale_index) |si| si else ScaleIndex.none, + } }; + } + + pub fn initRip(ptr_size: PtrSize, displacement: i32) Memory { + return .{ .rip = .{ .ptr_size = ptr_size, .disp = displacement } }; + } + + pub fn isSegmentRegister(mem: Memory) bool { + return switch (mem) { + .moffs => true, + .rip => false, + .sib => |s| switch (s.base) { + .none, .frame, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => false, + .reg => |reg| reg.isClass(.segment), + }, + }; + } + + pub fn base(mem: Memory) Base { + return switch (mem) { + .moffs => |m| .{ .reg = m.seg }, + .sib => |s| s.base, + .rip => .none, + }; + } + + pub fn scaleIndex(mem: Memory) ?ScaleIndex { + return switch (mem) { + .moffs, .rip => null, + .sib => |s| if (s.scale_index.scale > 0) s.scale_index else null, + }; + } + + pub fn disp(mem: Memory) Immediate { + return switch (mem) { + .sib => |s| .s(s.disp), + .rip => |r| .s(r.disp), + .moffs => |m| .u(m.offset), + }; + } + + pub fn bitSize(mem: Memory, target: *const std.Target) u64 { + return switch (mem) { + .rip => |r| r.ptr_size.bitSize(target), + .sib => |s| s.ptr_size.bitSize(target), + .moffs => target.ptrBitWidth(), + }; + } + }; + + pub const Operand = union(enum) { + none, + reg: Register, + mem: Memory, + imm: Immediate, + bytes: []const u8, + + /// Returns the bitsize of the operand. + pub fn bitSize(op: Operand) u64 { + return switch (op) { + .none => unreachable, + .reg => |reg| reg.bitSize(), + .mem => |mem| mem.bitSize(), + .imm => unreachable, + .bytes => unreachable, + }; + } + + /// Returns true if the operand is a segment register. + /// Asserts the operand is either register or memory. + pub fn isSegmentRegister(op: Operand) bool { + return switch (op) { + .none => unreachable, + .reg => |reg| reg.isClass(.segment), + .mem => |mem| mem.isSegmentRegister(), + .imm => unreachable, + .bytes => unreachable, + }; + } + + pub fn baseExtEnc(op: Operand) u2 { + return switch (op) { + .none, .imm => 0b00, + .reg => |reg| @truncate(reg.enc() >> 3), + .mem => |mem| switch (mem.base()) { + .none, .frame, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => 0b00, // rsp, rbp, and rip are not extended + .reg => |reg| @truncate(reg.enc() >> 3), + }, + .bytes => unreachable, + }; + } + + pub fn indexExtEnc(op: Operand) u2 { + return switch (op) { + .none, .reg, .imm => 0b00, + .mem => |mem| if (mem.scaleIndex()) |si| @truncate(si.index.enc() >> 3) else 0b00, + .bytes => unreachable, + }; + } + + const Format = struct { + op: Operand, + enc_op: Encoding.Op, + + fn default(f: Format, w: *Writer) Writer.Error!void { + const op = f.op; + const enc_op = f.enc_op; + switch (op) { + .none => {}, + .reg => |reg| try w.writeAll(@tagName(reg)), + .mem => |mem| switch (mem) { + .rip => |rip| { + try w.print("{f} [rip", .{rip.ptr_size}); + if (rip.disp != 0) try w.print(" {c} 0x{x}", .{ + @as(u8, if (rip.disp < 0) '-' else '+'), + @abs(rip.disp), + }); + try w.writeByte(']'); + }, + .sib => |sib| { + try w.print("{f} ", .{sib.ptr_size}); + + if (mem.isSegmentRegister()) { + return w.print("{s}:0x{x}", .{ @tagName(sib.base.reg), sib.disp }); + } + + try w.writeByte('['); + + var any = true; + switch (sib.base) { + .none => any = false, + .reg => |reg| try w.print("{s}", .{@tagName(reg)}), + .frame => |frame_index| try w.print("{f}", .{frame_index}), + .table => try w.print("Table", .{}), + .rip_inst => |inst_index| try w.print("RipInst({d})", .{inst_index}), + .nav => |nav| try w.print("Nav({d})", .{@intFromEnum(nav)}), + .uav => |uav| try w.print("Uav({d})", .{@intFromEnum(uav.val)}), + .lazy_sym => |lazy_sym| try w.print("LazySym({s}, {d})", .{ + @tagName(lazy_sym.kind), + @intFromEnum(lazy_sym.ty), + }), + .extern_func => |extern_func| try w.print("ExternFunc({d})", .{@intFromEnum(extern_func)}), + } + if (mem.scaleIndex()) |si| { + if (any) try w.writeAll(" + "); + try w.print("{s} * {d}", .{ @tagName(si.index), si.scale }); + any = true; + } + if (sib.disp != 0 or !any) { + if (any) + try w.print(" {c} ", .{@as(u8, if (sib.disp < 0) '-' else '+')}) + else if (sib.disp < 0) + try w.writeByte('-'); + try w.print("0x{x}", .{@abs(sib.disp)}); + any = true; + } + + try w.writeByte(']'); + }, + .moffs => |moffs| try w.print("{s}:0x{x}", .{ + @tagName(moffs.seg), + moffs.offset, + }), + }, + .imm => |imm| if (enc_op.isSigned()) { + const imms = imm.asSigned(enc_op.immBitSize()); + if (imms < 0) try w.writeByte('-'); + try w.print("0x{x}", .{@abs(imms)}); + } else try w.print("0x{x}", .{imm.asUnsigned(enc_op.immBitSize())}), + .bytes => unreachable, + } + } + }; + + pub fn fmt(op: Operand, enc_op: Encoding.Op) std.fmt.Alt(Format, Format.default) { + return .{ .data = .{ .op = op, .enc_op = enc_op } }; + } + }; + + pub fn new( + prefix: Prefix, + mnemonic: Mnemonic, + ops: []const Operand, + target: *const std.Target, + ) !Instruction { + const encoding: Encoding = switch (prefix) { + else => (try Encoding.findByMnemonic(prefix, mnemonic, ops, target)) orelse { + log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{ + @tagName(prefix), + @tagName(mnemonic), + @tagName(if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none), + @tagName(if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none), + @tagName(if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none), + @tagName(if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none), + }); + return error.InvalidInstruction; + }, + .directive => .{ + .mnemonic = mnemonic, + .data = .{ + .op_en = .z, + .ops = .{ + if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none, + if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none, + if (ops.len > 2) Encoding.Op.fromOperand(ops[2], target) else .none, + if (ops.len > 3) Encoding.Op.fromOperand(ops[3], target) else .none, + }, + .opc_len = 0, + .opc = undefined, + .modrm_ext = 0, + .mode = .none, + .feature = .none, + }, + }, + }; + log.debug("selected encoding: {f}", .{encoding}); + + var inst: Instruction = .{ + .prefix = prefix, + .encoding = encoding, + .ops = [1]Operand{.none} ** 4, + }; + @memcpy(inst.ops[0..ops.len], ops); + return inst; + } + + pub fn format(inst: Instruction, w: *Writer) Writer.Error!void { + switch (inst.prefix) { + .none, .directive => {}, + else => try w.print("{s} ", .{@tagName(inst.prefix)}), + } + try w.print("{s}", .{@tagName(inst.encoding.mnemonic)}); + for (inst.ops, inst.encoding.data.ops, 0..) |op, enc, i| { + if (op == .none) break; + if (i > 0) try w.writeByte(','); + try w.print(" {f}", .{op.fmt(enc)}); + } + } + + pub fn encode(inst: Instruction, w: *Writer, comptime opts: Options) !void { + assert(inst.prefix != .directive); + const encoder: Encoder(opts) = .{ .w = w }; + const enc = inst.encoding; + const data = enc.data; + + try inst.encodeWait(encoder); + if (data.mode.isVex()) { + try inst.encodeVexPrefix(encoder); + const opc = inst.encoding.opcode(); + try encoder.opcode_1byte(opc[opc.len - 1]); + } else { + try inst.encodeLegacyPrefixes(encoder); + try inst.encodeMandatoryPrefix(encoder); + try inst.encodeRexPrefix(encoder); + try inst.encodeOpcode(encoder); + } + + switch (data.op_en) { + .z, .o, .zo, .oz => {}, + .i, .d => try encodeImm(inst.ops[0].imm, data.ops[0], encoder), + .zi, .oi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder), + .ii => { + try encodeImm(inst.ops[0].imm, data.ops[0], encoder); + try encodeImm(inst.ops[1].imm, data.ops[1], encoder); + }, + .fd => try encoder.imm64(inst.ops[1].mem.moffs.offset), + .td => try encoder.imm64(inst.ops[0].mem.moffs.offset), + else => { + const mem_op: Operand = switch (data.op_en) { + .ia => .{ .reg = .eax }, + .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0], + .rm, .rmi, .rm0, .vm, .vmi, .rmv => inst.ops[1], + .rvm, .rvmr, .rvmi => inst.ops[2], + else => unreachable, + }; + switch (mem_op) { + .reg => |reg| { + const rm: u3 = switch (data.op_en) { + .ia, .m, .mi, .m1, .mc, .vm, .vmi => enc.modRmExt(), + .mr, .mri, .mrc => @truncate(inst.ops[1].reg.enc()), + .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => @truncate(inst.ops[0].reg.enc()), + .mvr => @truncate(inst.ops[2].reg.enc()), + else => unreachable, + }; + try encoder.modRm_direct(rm, @truncate(reg.enc())); + }, + .mem => |mem| { + const op = switch (data.op_en) { + .m, .mi, .m1, .mc, .vm, .vmi => .none, + .mr, .mri, .mrc => inst.ops[1], + .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0], + .mvr => inst.ops[2], + else => unreachable, + }; + try encodeMemory(enc, mem, op, encoder); + }, + else => unreachable, + } + + switch (data.op_en) { + .ia => try encodeImm(inst.ops[0].imm, data.ops[0], encoder), + .mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder), + .rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder), + .rvmr => try encoder.imm8(@as(u8, @as(u4, @intCast(inst.ops[3].reg.enc()))) << 4), + .rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder), + else => {}, + } + }, + } + } + + fn encodeOpcode(inst: Instruction, encoder: anytype) !void { + const opcode = inst.encoding.opcode(); + const first = @intFromBool(inst.encoding.mandatoryPrefix() != null); + const final = opcode.len - 1; + for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte); + switch (inst.encoding.data.op_en) { + .o, .oz, .oi => try encoder.opcode_withReg(opcode[final], @truncate(inst.ops[0].reg.enc())), + .zo => try encoder.opcode_withReg(opcode[final], @truncate(inst.ops[1].reg.enc())), + else => try encoder.opcode_1byte(opcode[final]), + } + } + + fn encodeWait(inst: Instruction, encoder: anytype) !void { + switch (inst.encoding.data.mode) { + .wait => try encoder.opcode_1byte(0x9b), + else => {}, + } + } + + fn encodeLegacyPrefixes(inst: Instruction, encoder: anytype) !void { + const enc = inst.encoding; + const data = enc.data; + const op_en = data.op_en; + + var legacy = LegacyPrefixes{}; + + switch (inst.prefix) { + .none => {}, + .lock => legacy.prefix_f0 = true, + .repne, .repnz => legacy.prefix_f2 = true, + .rep, .repe, .repz => legacy.prefix_f3 = true, + .directive => unreachable, + } + + switch (data.mode) { + .short, .rex_short => legacy.set16BitOverride(), + else => {}, + } + + const segment_override: ?Register = switch (op_en) { + .z, .i, .zi, .ii, .ia, .o, .zo, .oz, .oi, .d => null, + .fd => inst.ops[1].mem.base().reg, + .td => inst.ops[0].mem.base().reg, + .rm, .rmi, .rm0 => if (inst.ops[1].isSegmentRegister()) + switch (inst.ops[1]) { + .reg => |reg| reg, + .mem => |mem| mem.base().reg, + else => unreachable, + } + else + null, + .m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister()) + switch (inst.ops[0]) { + .reg => |reg| reg, + .mem => |mem| mem.base().reg, + else => unreachable, + } + else + null, + .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => unreachable, + }; + if (segment_override) |seg| { + legacy.setSegmentOverride(seg); + } + + try encoder.legacyPrefixes(legacy); + } + + fn encodeRexPrefix(inst: Instruction, encoder: anytype) !void { + const op_en = inst.encoding.data.op_en; + + var rex = Rex{}; + rex.present = inst.encoding.data.mode == .rex; + rex.w = inst.encoding.data.mode == .long; + + switch (op_en) { + .z, .i, .zi, .ii, .ia, .fd, .td, .d => {}, + .o, .oz, .oi => rex.b = inst.ops[0].reg.enc() & 0b01000 != 0, + .zo => rex.b = inst.ops[1].reg.enc() & 0b01000 != 0, + .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .rmv => { + const r_op = switch (op_en) { + .rm, .rmi, .rm0, .rmv => inst.ops[0], + .mr, .mri, .mrc => inst.ops[1], + else => .none, + }; + const r_op_base_ext_enc = r_op.baseExtEnc(); + rex.r = r_op_base_ext_enc & 0b01 != 0; + assert(r_op_base_ext_enc & 0b10 == 0); + + const b_x_op = switch (op_en) { + .rm, .rmi, .rm0 => inst.ops[1], + .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0], + else => unreachable, + }; + const b_x_op_base_ext_enc = b_x_op.baseExtEnc(); + rex.b = b_x_op_base_ext_enc & 0b01 != 0; + assert(b_x_op_base_ext_enc & 0b10 == 0); + const b_x_op_index_ext_enc = b_x_op.indexExtEnc(); + rex.x = b_x_op_index_ext_enc & 0b01 != 0; + assert(b_x_op_index_ext_enc & 0b10 == 0); + }, + .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable, + } + + try encoder.rex(rex); + } + + fn encodeVexPrefix(inst: Instruction, encoder: anytype) !void { + const op_en = inst.encoding.data.op_en; + const opc = inst.encoding.opcode(); + const mand_pre = inst.encoding.mandatoryPrefix(); + + var vex = Vex{}; + + vex.w = inst.encoding.data.mode.isLong(); + + switch (op_en) { + .z, .i, .zi, .ii, .ia, .fd, .td, .d, .o, .oz, .oi, .zo => unreachable, + .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vm, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => { + const r_op = switch (op_en) { + .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0], + .mr, .mri, .mrc => inst.ops[1], + .mvr => inst.ops[2], + .m, .mi, .m1, .mc, .vm, .vmi => .none, + else => unreachable, + }; + const r_op_base_ext_enc = r_op.baseExtEnc(); + vex.r = r_op_base_ext_enc & 0b01 != 0; + assert(r_op_base_ext_enc & 0b10 == 0); + + const b_x_op = switch (op_en) { + .rm, .rmi, .rm0, .vm, .vmi, .rmv => inst.ops[1], + .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0], + .rvm, .rvmr, .rvmi => inst.ops[2], + else => unreachable, + }; + const b_x_op_base_ext_enc = b_x_op.baseExtEnc(); + vex.b = b_x_op_base_ext_enc & 0b01 != 0; + assert(b_x_op_base_ext_enc & 0b10 == 0); + const b_x_op_index_ext_enc = b_x_op.indexExtEnc(); + vex.x = b_x_op_index_ext_enc & 0b01 != 0; + assert(b_x_op_index_ext_enc & 0b10 == 0); + }, + } + + vex.l = inst.encoding.data.mode.isVecLong(); + + vex.p = if (mand_pre) |mand| switch (mand) { + 0x66 => .@"66", + 0xf2 => .f2, + 0xf3 => .f3, + else => unreachable, + } else .none; + + const leading: usize = if (mand_pre) |_| 1 else 0; + assert(opc[leading] == 0x0f); + vex.m = switch (opc[leading + 1]) { + else => .@"0f", + 0x38 => .@"0f38", + 0x3a => .@"0f3a", + }; + + switch (op_en) { + else => {}, + .vm, .vmi => vex.v = inst.ops[0].reg, + .rvm, .rvmr, .rvmi => vex.v = inst.ops[1].reg, + .rmv => vex.v = inst.ops[2].reg, + } + + try encoder.vex(vex); + } + + fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void { + const prefix = inst.encoding.mandatoryPrefix() orelse return; + try encoder.opcode_1byte(prefix); + } + + fn encodeMemory(encoding: Encoding, mem: Memory, operand: Operand, encoder: anytype) !void { + const operand_enc: u3 = switch (operand) { + .reg => |reg| @truncate(reg.enc()), + .none => encoding.modRmExt(), + else => unreachable, + }; + + switch (mem) { + .moffs => unreachable, + .sib => |sib| switch (sib.base) { + .none, .table => { + try encoder.modRm_SIBDisp0(operand_enc); + if (mem.scaleIndex()) |si| { + const scale = math.log2_int(u4, si.scale); + try encoder.sib_scaleIndexDisp32(scale, @truncate(si.index.enc())); + } else { + try encoder.sib_disp32(); + } + try encoder.disp32(sib.disp); + }, + .reg => |base| switch (base.class()) { + .segment => { + // TODO audit this wrt SIB + try encoder.modRm_SIBDisp0(operand_enc); + if (mem.scaleIndex()) |si| { + const scale = math.log2_int(u4, si.scale); + try encoder.sib_scaleIndexDisp32(scale, @truncate(si.index.enc())); + } else { + try encoder.sib_disp32(); + } + try encoder.disp32(sib.disp); + }, + .general_purpose => { + const dst: u3 = @truncate(base.enc()); + const src = operand_enc; + if (dst == 4 or mem.scaleIndex() != null) { + if (sib.disp == 0 and dst != 5) { + try encoder.modRm_SIBDisp0(src); + if (mem.scaleIndex()) |si| { + const scale = math.log2_int(u4, si.scale); + try encoder.sib_scaleIndexBase(scale, @truncate(si.index.enc()), dst); + } else { + try encoder.sib_base(dst); + } + } else if (math.cast(i8, sib.disp)) |_| { + try encoder.modRm_SIBDisp8(src); + if (mem.scaleIndex()) |si| { + const scale = math.log2_int(u4, si.scale); + try encoder.sib_scaleIndexBaseDisp8(scale, @truncate(si.index.enc()), dst); + } else { + try encoder.sib_baseDisp8(dst); + } + try encoder.disp8(@as(i8, @truncate(sib.disp))); + } else { + try encoder.modRm_SIBDisp32(src); + if (mem.scaleIndex()) |si| { + const scale = math.log2_int(u4, si.scale); + try encoder.sib_scaleIndexBaseDisp32(scale, @truncate(si.index.enc()), dst); + } else { + try encoder.sib_baseDisp32(dst); + } + try encoder.disp32(sib.disp); + } + } else { + if (sib.disp == 0 and dst != 5) { + try encoder.modRm_indirectDisp0(src, dst); + } else if (math.cast(i8, sib.disp)) |_| { + try encoder.modRm_indirectDisp8(src, dst); + try encoder.disp8(@as(i8, @truncate(sib.disp))); + } else { + try encoder.modRm_indirectDisp32(src, dst); + try encoder.disp32(sib.disp); + } + } + }, + else => unreachable, + }, + .frame => if (@TypeOf(encoder).options.allow_frame_locs) { + try encoder.modRm_indirectDisp32(operand_enc, 0); + try encoder.disp32(undefined); + } else return error.CannotEncode, + .nav, .uav, .lazy_sym, .extern_func => if (@TypeOf(encoder).options.allow_symbols) { + try encoder.modRm_indirectDisp32(operand_enc, 0); + try encoder.disp32(undefined); + } else return error.CannotEncode, + .rip_inst => { + try encoder.modRm_RIPDisp32(operand_enc); + try encoder.disp32(sib.disp); + }, + }, + .rip => |rip| { + try encoder.modRm_RIPDisp32(operand_enc); + try encoder.disp32(rip.disp); + }, + } + } + + fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void { + const raw = imm.asUnsigned(kind.immBitSize()); + switch (kind.immBitSize()) { + 8 => try encoder.imm8(@as(u8, @intCast(raw))), + 16 => try encoder.imm16(@as(u16, @intCast(raw))), + 32 => try encoder.imm32(@as(u32, @intCast(raw))), + 64 => try encoder.imm64(raw), + else => unreachable, + } + } +}; + +pub const LegacyPrefixes = packed struct { + /// LOCK + prefix_f0: bool = false, + /// REPNZ, REPNE, REP, Scalar Double-precision + prefix_f2: bool = false, + /// REPZ, REPE, REP, Scalar Single-precision + prefix_f3: bool = false, + + /// CS segment override or Branch not taken + prefix_2e: bool = false, + /// SS segment override + prefix_36: bool = false, + /// ES segment override + prefix_26: bool = false, + /// FS segment override + prefix_64: bool = false, + /// GS segment override + prefix_65: bool = false, + + /// Branch taken + prefix_3e: bool = false, + + /// Address size override (enables 16 bit address size) + prefix_67: bool = false, + + /// Operand size override (enables 16 bit operation) + prefix_66: bool = false, + + padding: u5 = 0, + + pub fn setSegmentOverride(self: *LegacyPrefixes, reg: Register) void { + assert(reg.isClass(.segment)); + switch (reg) { + .cs => self.prefix_2e = true, + .ss => self.prefix_36 = true, + .es => self.prefix_26 = true, + .fs => self.prefix_64 = true, + .gs => self.prefix_65 = true, + .ds => {}, + else => unreachable, + } + } + + pub fn set16BitOverride(self: *LegacyPrefixes) void { + self.prefix_66 = true; + } +}; + +pub const Options = struct { allow_frame_locs: bool = false, allow_symbols: bool = false }; + +fn Encoder(comptime opts: Options) type { + return struct { + w: *Writer, + + const Self = @This(); + pub const options = opts; + + // -------- + // Prefixes + // -------- + + /// Encodes legacy prefixes + pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) !void { + if (@as(u16, @bitCast(prefixes)) != 0) { + // Hopefully this path isn't taken very often, so we'll do it the slow way for now + + // LOCK + if (prefixes.prefix_f0) try self.w.writeByte(0xf0); + // REPNZ, REPNE, REP, Scalar Double-precision + if (prefixes.prefix_f2) try self.w.writeByte(0xf2); + // REPZ, REPE, REP, Scalar Single-precision + if (prefixes.prefix_f3) try self.w.writeByte(0xf3); + + // CS segment override or Branch not taken + if (prefixes.prefix_2e) try self.w.writeByte(0x2e); + // DS segment override + if (prefixes.prefix_36) try self.w.writeByte(0x36); + // ES segment override + if (prefixes.prefix_26) try self.w.writeByte(0x26); + // FS segment override + if (prefixes.prefix_64) try self.w.writeByte(0x64); + // GS segment override + if (prefixes.prefix_65) try self.w.writeByte(0x65); + + // Branch taken + if (prefixes.prefix_3e) try self.w.writeByte(0x3e); + + // Operand size override + if (prefixes.prefix_66) try self.w.writeByte(0x66); + + // Address size override + if (prefixes.prefix_67) try self.w.writeByte(0x67); + } + } + + /// Use 16 bit operand size + /// + /// Note that this flag is overridden by REX.W, if both are present. + pub fn prefix16BitMode(self: Self) !void { + try self.w.writeByte(0x66); + } + + /// Encodes a REX prefix byte given all the fields + /// + /// Use this byte whenever you need 64 bit operation, + /// or one of reg, index, r/m, base, or opcode-reg might be extended. + /// + /// See struct `Rex` for a description of each field. + pub fn rex(self: Self, fields: Rex) !void { + if (!fields.present and !fields.isSet()) return; + + var byte: u8 = 0b0100_0000; + + if (fields.w) byte |= 0b1000; + if (fields.r) byte |= 0b0100; + if (fields.x) byte |= 0b0010; + if (fields.b) byte |= 0b0001; + + try self.w.writeByte(byte); + } + + /// Encodes a VEX prefix given all the fields + /// + /// See struct `Vex` for a description of each field. + pub fn vex(self: Self, fields: Vex) !void { + if (fields.is3Byte()) { + try self.w.writeByte(0b1100_0100); + + try self.w.writeByte( + @as(u8, ~@intFromBool(fields.r)) << 7 | + @as(u8, ~@intFromBool(fields.x)) << 6 | + @as(u8, ~@intFromBool(fields.b)) << 5 | + @as(u8, @intFromEnum(fields.m)) << 0, + ); + + try self.w.writeByte( + @as(u8, @intFromBool(fields.w)) << 7 | + @as(u8, ~@as(u4, @intCast(fields.v.enc()))) << 3 | + @as(u8, @intFromBool(fields.l)) << 2 | + @as(u8, @intFromEnum(fields.p)) << 0, + ); + } else { + try self.w.writeByte(0b1100_0101); + try self.w.writeByte( + @as(u8, ~@intFromBool(fields.r)) << 7 | + @as(u8, ~@as(u4, @intCast(fields.v.enc()))) << 3 | + @as(u8, @intFromBool(fields.l)) << 2 | + @as(u8, @intFromEnum(fields.p)) << 0, + ); + } + } + + // ------ + // Opcode + // ------ + + /// Encodes a 1 byte opcode + pub fn opcode_1byte(self: Self, opcode: u8) !void { + try self.w.writeByte(opcode); + } + + /// Encodes a 2 byte opcode + /// + /// e.g. IMUL has the opcode 0x0f 0xaf, so you use + /// + /// encoder.opcode_2byte(0x0f, 0xaf); + pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) !void { + try self.w.writeAll(&.{ prefix, opcode }); + } + + /// Encodes a 3 byte opcode + /// + /// e.g. MOVSD has the opcode 0xf2 0x0f 0x10 + /// + /// encoder.opcode_3byte(0xf2, 0x0f, 0x10); + pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) !void { + try self.w.writeAll(&.{ prefix_1, prefix_2, opcode }); + } + + /// Encodes a 1 byte opcode with a reg field + /// + /// Remember to add a REX prefix byte if reg is extended! + pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) !void { + assert(opcode & 0b111 == 0); + try self.w.writeByte(opcode | reg); + } + + // ------ + // ModR/M + // ------ + + /// Construct a ModR/M byte given all the fields + /// + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) !void { + try self.w.writeByte(@as(u8, mod) << 6 | @as(u8, reg_or_opx) << 3 | rm); + } + + /// Construct a ModR/M byte using direct r/m addressing + /// r/m effective address: r/m + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) !void { + try self.modRm(0b11, reg_or_opx, rm); + } + + /// Construct a ModR/M byte using indirect r/m addressing + /// r/m effective address: [r/m] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) !void { + assert(rm != 4 and rm != 5); + try self.modRm(0b00, reg_or_opx, rm); + } + + /// Construct a ModR/M byte using indirect SIB addressing + /// r/m effective address: [SIB] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) !void { + try self.modRm(0b00, reg_or_opx, 0b100); + } + + /// Construct a ModR/M byte using RIP-relative addressing + /// r/m effective address: [RIP + disp32] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) !void { + try self.modRm(0b00, reg_or_opx, 0b101); + } + + /// Construct a ModR/M byte using indirect r/m with a 8bit displacement + /// r/m effective address: [r/m + disp8] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) !void { + assert(rm != 4); + try self.modRm(0b01, reg_or_opx, rm); + } + + /// Construct a ModR/M byte using indirect SIB with a 8bit displacement + /// r/m effective address: [SIB + disp8] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) !void { + try self.modRm(0b01, reg_or_opx, 0b100); + } + + /// Construct a ModR/M byte using indirect r/m with a 32bit displacement + /// r/m effective address: [r/m + disp32] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) !void { + assert(rm != 4); + try self.modRm(0b10, reg_or_opx, rm); + } + + /// Construct a ModR/M byte using indirect SIB with a 32bit displacement + /// r/m effective address: [SIB + disp32] + /// + /// Note reg's effective address is always just reg for the ModR/M byte. + /// Remember to add a REX prefix byte if reg or rm are extended! + pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) !void { + try self.modRm(0b10, reg_or_opx, 0b100); + } + + // --- + // SIB + // --- + + /// Construct a SIB byte given all the fields + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib(self: Self, scale: u2, index: u3, base: u3) !void { + try self.w.writeByte(@as(u8, scale) << 6 | @as(u8, index) << 3 | base); + } + + /// Construct a SIB byte with scale * index + base, no frills. + /// r/m effective address: [base + scale * index] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) !void { + assert(base != 5); + + try self.sib(scale, index, base); + } + + /// Construct a SIB byte with scale * index + disp32 + /// r/m effective address: [scale * index + disp32] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) !void { + // scale is actually ignored + // index = 4 means no index if and only if we haven't extended the register + // TODO enforce this + // base = 5 means no base, if mod == 0. + try self.sib(scale, index, 5); + } + + /// Construct a SIB byte with just base + /// r/m effective address: [base] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_base(self: Self, base: u3) !void { + assert(base != 5); + + // scale is actually ignored + // index = 4 means no index + try self.sib(0, 4, base); + } + + /// Construct a SIB byte with just disp32 + /// r/m effective address: [disp32] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_disp32(self: Self) !void { + // scale is actually ignored + // index = 4 means no index + // base = 5 means no base, if mod == 0. + try self.sib(0, 4, 5); + } + + /// Construct a SIB byte with scale * index + base + disp8 + /// r/m effective address: [base + scale * index + disp8] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) !void { + try self.sib(scale, index, base); + } + + /// Construct a SIB byte with base + disp8, no index + /// r/m effective address: [base + disp8] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_baseDisp8(self: Self, base: u3) !void { + // scale is ignored + // index = 4 means no index + try self.sib(0, 4, base); + } + + /// Construct a SIB byte with scale * index + base + disp32 + /// r/m effective address: [base + scale * index + disp32] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) !void { + try self.sib(scale, index, base); + } + + /// Construct a SIB byte with base + disp32, no index + /// r/m effective address: [base + disp32] + /// + /// Remember to add a REX prefix byte if index or base are extended! + pub fn sib_baseDisp32(self: Self, base: u3) !void { + // scale is ignored + // index = 4 means no index + try self.sib(0, 4, base); + } + + // ------------------------- + // Trivial (no bit fiddling) + // ------------------------- + + /// Encode an 8 bit displacement + /// + /// It is sign-extended to 64 bits by the cpu. + pub fn disp8(self: Self, disp: i8) !void { + try self.w.writeByte(@as(u8, @bitCast(disp))); + } + + /// Encode an 32 bit displacement + /// + /// It is sign-extended to 64 bits by the cpu. + pub fn disp32(self: Self, disp: i32) !void { + try self.w.writeInt(i32, disp, .little); + } + + /// Encode an 8 bit immediate + /// + /// It is sign-extended to 64 bits by the cpu. + pub fn imm8(self: Self, imm: u8) !void { + try self.w.writeByte(imm); + } + + /// Encode an 16 bit immediate + /// + /// It is sign-extended to 64 bits by the cpu. + pub fn imm16(self: Self, imm: u16) !void { + try self.w.writeInt(u16, imm, .little); + } + + /// Encode an 32 bit immediate + /// + /// It is sign-extended to 64 bits by the cpu. + pub fn imm32(self: Self, imm: u32) !void { + try self.w.writeInt(u32, imm, .little); + } + + /// Encode an 64 bit immediate + /// + /// It is sign-extended to 64 bits by the cpu. + pub fn imm64(self: Self, imm: u64) !void { + try self.w.writeInt(u64, imm, .little); + } + }; +} + +pub const Rex = struct { + w: bool = false, + r: bool = false, + x: bool = false, + b: bool = false, + present: bool = false, + + pub fn isSet(rex: Rex) bool { + return rex.w or rex.r or rex.x or rex.b; + } +}; + +pub const Vex = struct { + w: bool = false, + r: bool = false, + x: bool = false, + b: bool = false, + l: bool = false, + p: enum(u2) { + none = 0b00, + @"66" = 0b01, + f3 = 0b10, + f2 = 0b11, + } = .none, + m: enum(u5) { + @"0f" = 0b0_0001, + @"0f38" = 0b0_0010, + @"0f3a" = 0b0_0011, + _, + } = .@"0f", + v: Register = .ymm0, + + pub fn is3Byte(vex: Vex) bool { + return vex.w or vex.x or vex.b or vex.m != .@"0f"; + } +}; + +// Tests +fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void { + assert(expected.len > 0); + if (std.mem.eql(u8, expected, given)) return; + const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{expected}); + defer testing.allocator.free(expected_fmt); + const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{given}); + defer testing.allocator.free(given_fmt); + const idx = std.mem.indexOfDiff(u8, expected_fmt, given_fmt).?; + const padding = try testing.allocator.alloc(u8, idx + 5); + defer testing.allocator.free(padding); + @memset(padding, ' '); + std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{ + assembly, + expected_fmt, + given_fmt, + padding, + }); + return error.TestFailed; +} + +const TestEncode = struct { + buffer: [32]u8 = undefined, + index: usize = 0, + + fn encode( + enc: *TestEncode, + mnemonic: Instruction.Mnemonic, + ops: []const Instruction.Operand, + ) !void { + var writer: std.Io.Writer = .fixed(&enc.buffer); + const inst: Instruction = try .new(.none, mnemonic, ops); + try inst.encode(&writer, .{}); + enc.index = writer.bufferedLen(); + } + + fn code(enc: TestEncode) []const u8 { + return enc.buffer[0..enc.index]; + } +}; + +test "encode" { + var buf = std.array_list.Managed(u8).init(testing.allocator); + defer buf.deinit(); + + const inst: Instruction = try .new(.none, .mov, &.{ + .{ .reg = .rbx }, + .{ .imm = .u(4) }, + }); + try inst.encode(buf.writer(), .{}); + try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items); +} + +test "lower I encoding" { + var enc = TestEncode{}; + + try enc.encode(.push, &.{ + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10"); + + try enc.encode(.push, &.{ + .{ .imm = .u(0x1000) }, + }); + try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000"); + + try enc.encode(.push, &.{ + .{ .imm = .u(0x10000000) }, + }); + try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000"); + + try enc.encode(.adc, &.{ + .{ .reg = .rax }, + .{ .imm = .u(0x10000000) }, + }); + try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000"); + + try enc.encode(.add, &.{ + .{ .reg = .al }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10"); + + try enc.encode(.add, &.{ + .{ .reg = .rax }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); + + try enc.encode(.sbb, &.{ + .{ .reg = .ax }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10"); + + try enc.encode(.xor, &.{ + .{ .reg = .al }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10"); +} + +test "lower MI encoding" { + var enc = TestEncode{}; + + try enc.encode(.mov, &.{ + .{ .reg = .r12 }, + .{ .imm = .u(0x1000) }, + }); + try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .r12 } }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10"); + + try enc.encode(.mov, &.{ + .{ .reg = .r12 }, + .{ .imm = .u(0x1000) }, + }); + try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); + + try enc.encode(.mov, &.{ + .{ .reg = .r12 }, + .{ .imm = .u(0x1000) }, + }); + try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 } }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initRip(.qword, 0x10) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings( + "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00", + enc.code(), + "mov QWORD PTR [rip + 0x10], 0x10", + ); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -8 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -2 }) }, + .{ .imm = .s(-16) }, + }); + try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -1 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .{ .reg = .ds }, + .disp = 0x10000000, + .scale_index = .{ .scale = 2, .index = .rcx }, + }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings( + "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00", + enc.code(), + "mov QWORD PTR [rcx*2 + 0x10000000], 0x10", + ); + + try enc.encode(.adc, &.{ + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10"); + + try enc.encode(.adc, &.{ + .{ .mem = Instruction.Memory.initRip(.qword, 0) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10"); + + try enc.encode(.adc, &.{ + .{ .reg = .rax }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10"); + + try enc.encode(.add, &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .rdx }, .disp = -8 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10"); + + try enc.encode(.add, &.{ + .{ .reg = .rax }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); + + try enc.encode(.add, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, + .{ .imm = .s(-0x10) }, + }); + try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10"); + + try enc.encode(.@"and", &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings( + "\x83\x24\x25\x00\x00\x00\x10\x10", + enc.code(), + "and DWORD PTR ds:0x10000000, 0x10", + ); + + try enc.encode(.@"and", &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .es }, .disp = 0x10000000 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings( + "\x26\x83\x24\x25\x00\x00\x00\x10\x10", + enc.code(), + "and DWORD PTR es:0x10000000, 0x10", + ); + + try enc.encode(.@"and", &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings( + "\x41\x83\xA4\x24\x00\x00\x00\x10\x10", + enc.code(), + "and DWORD PTR [r12 + 0x10000000], 0x10", + ); + + try enc.encode(.sub, &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings( + "\x41\x83\xAB\x00\x00\x00\x10\x10", + enc.code(), + "sub DWORD PTR [r11 + 0x10000000], 0x10", + ); +} + +test "lower RM encoding" { + var enc = TestEncode{}; + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r11 } }) }, + }); + try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]"); + + try enc.encode(.mov, &.{ + .{ .reg = .rbx }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0x10 }) }, + }); + try expectEqualHexStrings("\x48\x8B\x1C\x25\x10\x00\x00\x00", enc.code(), "mov rbx, QWORD PTR ds:0x10"); + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -4 }) }, + }); + try expectEqualHexStrings("\x48\x8B\x45\xFC", enc.code(), "mov rax, QWORD PTR [rbp - 4]"); + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .{ .reg = .rbp }, + .scale_index = .{ .scale = 1, .index = .rcx }, + .disp = -8, + }) }, + }); + try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*1 - 8]"); + + try enc.encode(.mov, &.{ + .{ .reg = .eax }, + .{ .mem = Instruction.Memory.initSib(.dword, .{ + .base = .{ .reg = .rbp }, + .scale_index = .{ .scale = 4, .index = .rdx }, + .disp = -4, + }) }, + }); + try expectEqualHexStrings("\x8B\x44\x95\xFC", enc.code(), "mov eax, dword ptr [rbp + rdx*4 - 4]"); + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .{ .reg = .rbp }, + .scale_index = .{ .scale = 8, .index = .rcx }, + .disp = -8, + }) }, + }); + try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*8 - 8]"); + + try enc.encode(.mov, &.{ + .{ .reg = .r8b }, + .{ .mem = Instruction.Memory.initSib(.byte, .{ + .base = .{ .reg = .rsi }, + .scale_index = .{ .scale = 1, .index = .rcx }, + .disp = -24, + }) }, + }); + try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", enc.code(), "mov r8b, BYTE PTR [rsi + rcx*1 - 24]"); + + // TODO this mnemonic needs cleanup as some prefixes are obsolete. + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .reg = .cs }, + }); + try expectEqualHexStrings("\x48\x8C\xC8", enc.code(), "mov rax, cs"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, + .{ .reg = .fs }, + }); + try expectEqualHexStrings("\x8C\x65\xF0", enc.code(), "mov WORD PTR [rbp - 16], fs"); + + try enc.encode(.mov, &.{ + .{ .reg = .r12w }, + .{ .reg = .cs }, + }); + try expectEqualHexStrings("\x66\x41\x8C\xCC", enc.code(), "mov r12w, cs"); + + try enc.encode(.movsx, &.{ + .{ .reg = .eax }, + .{ .reg = .bx }, + }); + try expectEqualHexStrings("\x0F\xBF\xC3", enc.code(), "movsx eax, bx"); + + try enc.encode(.movsx, &.{ + .{ .reg = .eax }, + .{ .reg = .bl }, + }); + try expectEqualHexStrings("\x0F\xBE\xC3", enc.code(), "movsx eax, bl"); + + try enc.encode(.movsx, &.{ + .{ .reg = .ax }, + .{ .reg = .bl }, + }); + try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl"); + + try enc.encode(.movsx, &.{ + .{ .reg = .eax }, + .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp } }) }, + }); + try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]"); + + try enc.encode(.movsx, &.{ + .{ .reg = .eax }, + .{ .mem = Instruction.Memory.initSib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) }, + }); + try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]"); + + try enc.encode(.movsx, &.{ + .{ .reg = .ax }, + .{ .mem = Instruction.Memory.initRip(.byte, 0x10) }, + }); + try expectEqualHexStrings("\x66\x0F\xBE\x05\x10\x00\x00\x00", enc.code(), "movsx ax, BYTE PTR [rip + 0x10]"); + + try enc.encode(.movsx, &.{ + .{ .reg = .rax }, + .{ .reg = .bx }, + }); + try expectEqualHexStrings("\x48\x0F\xBF\xC3", enc.code(), "movsx rax, bx"); + + try enc.encode(.movsxd, &.{ + .{ .reg = .rax }, + .{ .reg = .ebx }, + }); + try expectEqualHexStrings("\x48\x63\xC3", enc.code(), "movsxd rax, ebx"); + + try enc.encode(.lea, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initRip(.qword, 0x10) }, + }); + try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, QWORD PTR [rip + 0x10]"); + + try enc.encode(.lea, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initRip(.dword, 0x10) }, + }); + try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, DWORD PTR [rip + 0x10]"); + + try enc.encode(.lea, &.{ + .{ .reg = .eax }, + .{ .mem = Instruction.Memory.initRip(.dword, 0x10) }, + }); + try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, DWORD PTR [rip + 0x10]"); + + try enc.encode(.lea, &.{ + .{ .reg = .eax }, + .{ .mem = Instruction.Memory.initRip(.word, 0x10) }, + }); + try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, WORD PTR [rip + 0x10]"); + + try enc.encode(.lea, &.{ + .{ .reg = .ax }, + .{ .mem = Instruction.Memory.initRip(.byte, 0x10) }, + }); + try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]"); + + try enc.encode(.lea, &.{ + .{ .reg = .rsi }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .{ .reg = .rbp }, + .scale_index = .{ .scale = 1, .index = .rcx }, + }) }, + }); + try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]"); + + try enc.encode(.add, &.{ + .{ .reg = .r11 }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, + }); + try expectEqualHexStrings("\x4C\x03\x1C\x25\x00\x00\x00\x10", enc.code(), "add r11, QWORD PTR ds:0x10000000"); + + try enc.encode(.add, &.{ + .{ .reg = .r12b }, + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, + }); + try expectEqualHexStrings("\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR ds:0x10000000"); + + try enc.encode(.add, &.{ + .{ .reg = .r12b }, + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .fs }, .disp = 0x10000000 }) }, + }); + try expectEqualHexStrings("\x64\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR fs:0x10000000"); + + try enc.encode(.sub, &.{ + .{ .reg = .r11 }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r13 }, .disp = 0x10000000 }) }, + }); + try expectEqualHexStrings("\x4D\x2B\x9D\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r13 + 0x10000000]"); + + try enc.encode(.sub, &.{ + .{ .reg = .r11 }, + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) }, + }); + try expectEqualHexStrings("\x4D\x2B\x9C\x24\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r12 + 0x10000000]"); + + try enc.encode(.imul, &.{ + .{ .reg = .r11 }, + .{ .reg = .r12 }, + }); + try expectEqualHexStrings("\x4D\x0F\xAF\xDC", enc.code(), "mov r11, r12"); +} + +test "lower RMI encoding" { + var enc = TestEncode{}; + + try enc.encode(.imul, &.{ + .{ .reg = .r11 }, + .{ .reg = .r12 }, + .{ .imm = .s(-2) }, + }); + try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2"); + + try enc.encode(.imul, &.{ + .{ .reg = .r11 }, + .{ .mem = Instruction.Memory.initRip(.qword, -16) }, + .{ .imm = .s(-1024) }, + }); + try expectEqualHexStrings( + "\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF", + enc.code(), + "imul r11, QWORD PTR [rip - 16], -1024", + ); + + try enc.encode(.imul, &.{ + .{ .reg = .bx }, + .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, + .{ .imm = .s(-1024) }, + }); + try expectEqualHexStrings( + "\x66\x69\x5D\xF0\x00\xFC", + enc.code(), + "imul bx, WORD PTR [rbp - 16], -1024", + ); + + try enc.encode(.imul, &.{ + .{ .reg = .bx }, + .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, + .{ .imm = .u(1024) }, + }); + try expectEqualHexStrings( + "\x66\x69\x5D\xF0\x00\x04", + enc.code(), + "imul bx, WORD PTR [rbp - 16], 1024", + ); +} + +test "lower MR encoding" { + var enc = TestEncode{}; + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .reg = .rbx }, + }); + try expectEqualHexStrings("\x48\x89\xD8", enc.code(), "mov rax, rbx"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -4 }) }, + .{ .reg = .r11 }, + }); + try expectEqualHexStrings("\x4c\x89\x5d\xfc", enc.code(), "mov QWORD PTR [rbp - 4], r11"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initRip(.qword, 0x10) }, + .{ .reg = .r12 }, + }); + try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rip + 0x10], r12"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .{ .reg = .r11 }, + .scale_index = .{ .scale = 2, .index = .r12 }, + .disp = 0x10, + }) }, + .{ .reg = .r13 }, + }); + try expectEqualHexStrings("\x4F\x89\x6C\x63\x10", enc.code(), "mov QWORD PTR [r11 + 2 * r12 + 0x10], r13"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initRip(.word, -0x10) }, + .{ .reg = .r12w }, + }); + try expectEqualHexStrings("\x66\x44\x89\x25\xF0\xFF\xFF\xFF", enc.code(), "mov WORD PTR [rip - 0x10], r12w"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initSib(.byte, .{ + .base = .{ .reg = .r11 }, + .scale_index = .{ .scale = 2, .index = .r12 }, + .disp = 0x10, + }) }, + .{ .reg = .r13b }, + }); + try expectEqualHexStrings("\x47\x88\x6C\x63\x10", enc.code(), "mov BYTE PTR [r11 + 2 * r12 + 0x10], r13b"); + + try enc.encode(.add, &.{ + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, + .{ .reg = .r12b }, + }); + try expectEqualHexStrings("\x44\x00\x24\x25\x00\x00\x00\x10", enc.code(), "add BYTE PTR ds:0x10000000, r12b"); + + try enc.encode(.add, &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, + .{ .reg = .r12d }, + }); + try expectEqualHexStrings("\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [ds:0x10000000], r12d"); + + try enc.encode(.add, &.{ + .{ .mem = Instruction.Memory.initSib(.dword, .{ .base = .{ .reg = .gs }, .disp = 0x10000000 }) }, + .{ .reg = .r12d }, + }); + try expectEqualHexStrings("\x65\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [gs:0x10000000], r12d"); + + try enc.encode(.sub, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) }, + .{ .reg = .r12 }, + }); + try expectEqualHexStrings("\x4D\x29\xA3\x00\x00\x00\x10", enc.code(), "sub QWORD PTR [r11 + 0x10000000], r12"); +} + +test "lower M encoding" { + var enc = TestEncode{}; + + try enc.encode(.call, &.{ + .{ .reg = .r12 }, + }); + try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12"); + + try enc.encode(.call, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .r12 } }) }, + }); + try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]"); + + try enc.encode(.call, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .none, + .scale_index = .{ .index = .r11, .scale = 2 }, + }) }, + }); + try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]"); + + try enc.encode(.call, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ + .base = .none, + .scale_index = .{ .index = .r12, .scale = 2 }, + }) }, + }); + try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]"); + + try enc.encode(.call, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .gs } }) }, + }); + try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0"); + + try enc.encode(.call, &.{ + .{ .imm = .s(0) }, + }); + try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0"); + + try enc.encode(.push, &.{ + .{ .mem = Instruction.Memory.initSib(.qword, .{ .base = .{ .reg = .rbp } }) }, + }); + try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]"); + + try enc.encode(.push, &.{ + .{ .mem = Instruction.Memory.initSib(.word, .{ .base = .{ .reg = .rbp } }) }, + }); + try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]"); + + try enc.encode(.pop, &.{ + .{ .mem = Instruction.Memory.initRip(.qword, 0) }, + }); + try expectEqualHexStrings("\x8F\x05\x00\x00\x00\x00", enc.code(), "pop QWORD PTR [rip]"); + + try enc.encode(.pop, &.{ + .{ .mem = Instruction.Memory.initRip(.word, 0) }, + }); + try expectEqualHexStrings("\x66\x8F\x05\x00\x00\x00\x00", enc.code(), "pop WORD PTR [rbp]"); + + try enc.encode(.imul, &.{ + .{ .reg = .rax }, + }); + try expectEqualHexStrings("\x48\xF7\xE8", enc.code(), "imul rax"); + + try enc.encode(.imul, &.{ + .{ .reg = .r12 }, + }); + try expectEqualHexStrings("\x49\xF7\xEC", enc.code(), "imul r12"); +} + +test "lower O encoding" { + var enc = TestEncode{}; + + try enc.encode(.push, &.{ + .{ .reg = .rax }, + }); + try expectEqualHexStrings("\x50", enc.code(), "push rax"); + + try enc.encode(.push, &.{ + .{ .reg = .r12w }, + }); + try expectEqualHexStrings("\x66\x41\x54", enc.code(), "push r12w"); + + try enc.encode(.pop, &.{ + .{ .reg = .r12 }, + }); + try expectEqualHexStrings("\x41\x5c", enc.code(), "pop r12"); +} + +test "lower OI encoding" { + var enc = TestEncode{}; + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .imm = .u(0x1000000000000000) }, + }); + try expectEqualHexStrings( + "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", + enc.code(), + "movabs rax, 0x1000000000000000", + ); + + try enc.encode(.mov, &.{ + .{ .reg = .r11 }, + .{ .imm = .u(0x1000000000000000) }, + }); + try expectEqualHexStrings( + "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", + enc.code(), + "movabs r11, 0x1000000000000000", + ); + + try enc.encode(.mov, &.{ + .{ .reg = .r11d }, + .{ .imm = .u(0x10000000) }, + }); + try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000"); + + try enc.encode(.mov, &.{ + .{ .reg = .r11w }, + .{ .imm = .u(0x1000) }, + }); + try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000"); + + try enc.encode(.mov, &.{ + .{ .reg = .r11b }, + .{ .imm = .u(0x10) }, + }); + try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10"); +} + +test "lower FD/TD encoding" { + var enc = TestEncode{}; + + try enc.encode(.mov, &.{ + .{ .reg = .rax }, + .{ .mem = Instruction.Memory.initMoffs(.cs, 0x10) }, + }); + try expectEqualHexStrings("\x2E\x48\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs rax, cs:0x10"); + + try enc.encode(.mov, &.{ + .{ .reg = .eax }, + .{ .mem = Instruction.Memory.initMoffs(.fs, 0x10) }, + }); + try expectEqualHexStrings("\x64\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs eax, fs:0x10"); + + try enc.encode(.mov, &.{ + .{ .reg = .ax }, + .{ .mem = Instruction.Memory.initMoffs(.gs, 0x10) }, + }); + try expectEqualHexStrings("\x65\x66\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ax, gs:0x10"); + + try enc.encode(.mov, &.{ + .{ .reg = .al }, + .{ .mem = Instruction.Memory.initMoffs(.ds, 0x10) }, + }); + try expectEqualHexStrings("\xA0\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs al, ds:0x10"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initMoffs(.cs, 0x10) }, + .{ .reg = .rax }, + }); + try expectEqualHexStrings("\x2E\x48\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs cs:0x10, rax"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initMoffs(.fs, 0x10) }, + .{ .reg = .eax }, + }); + try expectEqualHexStrings("\x64\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs fs:0x10, eax"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initMoffs(.gs, 0x10) }, + .{ .reg = .ax }, + }); + try expectEqualHexStrings("\x65\x66\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs gs:0x10, ax"); + + try enc.encode(.mov, &.{ + .{ .mem = Instruction.Memory.initMoffs(.ds, 0x10) }, + .{ .reg = .al }, + }); + try expectEqualHexStrings("\xA2\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ds:0x10, al"); +} + +test "lower NP encoding" { + var enc = TestEncode{}; + + try enc.encode(.int3, &.{}); + try expectEqualHexStrings("\xCC", enc.code(), "int3"); + + try enc.encode(.nop, &.{}); + try expectEqualHexStrings("\x90", enc.code(), "nop"); + + try enc.encode(.ret, &.{}); + try expectEqualHexStrings("\xC3", enc.code(), "ret"); + + try enc.encode(.syscall, &.{}); + try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall"); +} + +fn invalidInstruction(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void { + const err: Instruction = .new(.none, mnemonic, ops); + try testing.expectError(error.InvalidInstruction, err); +} + +test "invalid instruction" { + try invalidInstruction(.call, &.{ + .{ .reg = .eax }, + }); + try invalidInstruction(.call, &.{ + .{ .reg = .ax }, + }); + try invalidInstruction(.call, &.{ + .{ .reg = .al }, + }); + try invalidInstruction(.call, &.{ + .{ .mem = Instruction.Memory.initRip(.dword, 0) }, + }); + try invalidInstruction(.call, &.{ + .{ .mem = Instruction.Memory.initRip(.word, 0) }, + }); + try invalidInstruction(.call, &.{ + .{ .mem = Instruction.Memory.initRip(.byte, 0) }, + }); + try invalidInstruction(.mov, &.{ + .{ .mem = Instruction.Memory.initRip(.word, 0x10) }, + .{ .reg = .r12 }, + }); + try invalidInstruction(.lea, &.{ + .{ .reg = .rax }, + .{ .reg = .rbx }, + }); + try invalidInstruction(.lea, &.{ + .{ .reg = .al }, + .{ .mem = Instruction.Memory.initRip(.byte, 0) }, + }); + try invalidInstruction(.pop, &.{ + .{ .reg = .r12b }, + }); + try invalidInstruction(.pop, &.{ + .{ .reg = .r12d }, + }); + try invalidInstruction(.push, &.{ + .{ .reg = .r12b }, + }); + try invalidInstruction(.push, &.{ + .{ .reg = .r12d }, + }); + try invalidInstruction(.push, &.{ + .{ .imm = .u(0x1000000000000000) }, + }); +} + +fn cannotEncode(mnemonic: Instruction.Mnemonic, ops: []const Instruction.Operand) !void { + try testing.expectError(error.CannotEncode, .new(.none, mnemonic, ops)); +} + +test "cannot encode" { + try cannotEncode(.@"test", &.{ + .{ .mem = Instruction.Memory.initSib(.byte, .{ .base = .{ .reg = .r12 } }) }, + .{ .reg = .ah }, + }); + try cannotEncode(.@"test", &.{ + .{ .reg = .r11b }, + .{ .reg = .bh }, + }); + try cannotEncode(.mov, &.{ + .{ .reg = .sil }, + .{ .reg = .ah }, + }); +} + +const Assembler = struct { + it: Tokenizer, + + const Tokenizer = struct { + input: []const u8, + pos: usize = 0, + + const Error = error{InvalidToken}; + + const Token = struct { + id: Id, + start: usize, + end: usize, + + const Id = enum { + eof, + + space, + new_line, + + colon, + comma, + open_br, + close_br, + plus, + minus, + star, + + string, + numeral, + }; + }; + + const Iterator = struct {}; + + fn next(it: *Tokenizer) !Token { + var result = Token{ + .id = .eof, + .start = it.pos, + .end = it.pos, + }; + + var state: enum { + start, + space, + new_line, + string, + numeral, + numeral_hex, + } = .start; + + while (it.pos < it.input.len) : (it.pos += 1) { + const ch = it.input[it.pos]; + switch (state) { + .start => switch (ch) { + ',' => { + result.id = .comma; + it.pos += 1; + break; + }, + ':' => { + result.id = .colon; + it.pos += 1; + break; + }, + '[' => { + result.id = .open_br; + it.pos += 1; + break; + }, + ']' => { + result.id = .close_br; + it.pos += 1; + break; + }, + '+' => { + result.id = .plus; + it.pos += 1; + break; + }, + '-' => { + result.id = .minus; + it.pos += 1; + break; + }, + '*' => { + result.id = .star; + it.pos += 1; + break; + }, + ' ', '\t' => state = .space, + '\n', '\r' => state = .new_line, + 'a'...'z', 'A'...'Z' => state = .string, + '0'...'9' => state = .numeral, + else => return error.InvalidToken, + }, + + .space => switch (ch) { + ' ', '\t' => {}, + else => { + result.id = .space; + break; + }, + }, + + .new_line => switch (ch) { + '\n', '\r', ' ', '\t' => {}, + else => { + result.id = .new_line; + break; + }, + }, + + .string => switch (ch) { + 'a'...'z', 'A'...'Z', '0'...'9' => {}, + else => { + result.id = .string; + break; + }, + }, + + .numeral => switch (ch) { + 'x' => state = .numeral_hex, + '0'...'9' => {}, + else => { + result.id = .numeral; + break; + }, + }, + + .numeral_hex => switch (ch) { + 'a'...'f' => {}, + '0'...'9' => {}, + else => { + result.id = .numeral; + break; + }, + }, + } + } + + if (it.pos >= it.input.len) { + switch (state) { + .string => result.id = .string, + .numeral, .numeral_hex => result.id = .numeral, + else => {}, + } + } + + result.end = it.pos; + return result; + } + + fn seekTo(it: *Tokenizer, pos: usize) void { + it.pos = pos; + } + }; + + pub fn init(input: []const u8) Assembler { + return .{ + .it = Tokenizer{ .input = input }, + }; + } + + pub fn assemble(as: *Assembler, w: *Writer) !void { + while (try as.next()) |parsed_inst| { + const inst: Instruction = try .new(.none, parsed_inst.mnemonic, &parsed_inst.ops); + try inst.encode(w, .{}); + } + } + + const ParseResult = struct { + mnemonic: Instruction.Mnemonic, + ops: [4]Instruction.Operand, + }; + + const ParseError = error{ + UnexpectedToken, + InvalidMnemonic, + InvalidOperand, + InvalidRegister, + InvalidPtrSize, + InvalidMemoryOperand, + InvalidScaleIndex, + } || Tokenizer.Error || std.fmt.ParseIntError; + + fn next(as: *Assembler) ParseError!?ParseResult { + try as.skip(2, .{ .space, .new_line }); + const mnemonic_tok = as.expect(.string) catch |err| switch (err) { + error.UnexpectedToken => return if (try as.peek() == .eof) null else err, + else => return err, + }; + const mnemonic = mnemonicFromString(as.source(mnemonic_tok)) orelse + return error.InvalidMnemonic; + try as.skip(1, .{.space}); + + const rules = .{ + .{}, + .{.register}, + .{.memory}, + .{.immediate}, + .{ .register, .register }, + .{ .register, .memory }, + .{ .memory, .register }, + .{ .register, .immediate }, + .{ .memory, .immediate }, + .{ .register, .register, .immediate }, + .{ .register, .memory, .immediate }, + }; + + const pos = as.it.pos; + inline for (rules) |rule| { + var ops = [4]Instruction.Operand{ .none, .none, .none, .none }; + if (as.parseOperandRule(rule, &ops)) { + return .{ + .mnemonic = mnemonic, + .ops = ops, + }; + } else |_| { + as.it.seekTo(pos); + } + } + + return error.InvalidOperand; + } + + fn source(as: *Assembler, token: Tokenizer.Token) []const u8 { + return as.it.input[token.start..token.end]; + } + + fn peek(as: *Assembler) Tokenizer.Error!Tokenizer.Token.Id { + const pos = as.it.pos; + const next_tok = try as.it.next(); + const id = next_tok.id; + as.it.seekTo(pos); + return id; + } + + fn expect(as: *Assembler, id: Tokenizer.Token.Id) ParseError!Tokenizer.Token { + const next_tok_id = try as.peek(); + if (next_tok_id == id) return as.it.next(); + return error.UnexpectedToken; + } + + fn skip(as: *Assembler, comptime num: comptime_int, tok_ids: [num]Tokenizer.Token.Id) Tokenizer.Error!void { + outer: while (true) { + const pos = as.it.pos; + const next_tok = try as.it.next(); + inline for (tok_ids) |tok_id| { + if (next_tok.id == tok_id) continue :outer; + } + as.it.seekTo(pos); + break; + } + } + + fn mnemonicFromString(bytes: []const u8) ?Instruction.Mnemonic { + const ti = @typeInfo(Instruction.Mnemonic).@"enum"; + inline for (ti.fields) |field| { + if (std.mem.eql(u8, bytes, field.name)) { + return @field(Instruction.Mnemonic, field.name); + } + } + return null; + } + + fn parseOperandRule(as: *Assembler, rule: anytype, ops: *[4]Instruction.Operand) ParseError!void { + inline for (rule, 0..) |cond, i| { + comptime assert(i < 4); + if (i > 0) { + _ = try as.expect(.comma); + try as.skip(1, .{.space}); + } + if (@typeInfo(@TypeOf(cond)) != .enum_literal) { + @compileError("invalid condition in the rule: " ++ @typeName(@TypeOf(cond))); + } + switch (cond) { + .register => { + const reg_tok = try as.expect(.string); + const reg = registerFromString(as.source(reg_tok)) orelse + return error.InvalidOperand; + ops[i] = .{ .reg = reg }; + }, + .memory => { + const mem = try as.parseMemory(); + ops[i] = .{ .mem = mem }; + }, + .immediate => { + const is_neg = if (as.expect(.minus)) |_| true else |_| false; + const imm_tok = try as.expect(.numeral); + const imm: Instruction.Immediate = if (is_neg) blk: { + const imm = try std.fmt.parseInt(i32, as.source(imm_tok), 0); + break :blk .{ .signed = imm * -1 }; + } else .{ .unsigned = try std.fmt.parseInt(u64, as.source(imm_tok), 0) }; + ops[i] = .{ .imm = imm }; + }, + else => @compileError("unhandled enum literal " ++ @tagName(cond)), + } + try as.skip(1, .{.space}); + } + + try as.skip(1, .{.space}); + const tok = try as.it.next(); + switch (tok.id) { + .new_line, .eof => {}, + else => return error.InvalidOperand, + } + } + + fn registerFromString(bytes: []const u8) ?Register { + const ti = @typeInfo(Register).@"enum"; + inline for (ti.fields) |field| { + if (std.mem.eql(u8, bytes, field.name)) { + return @field(Register, field.name); + } + } + return null; + } + + fn parseMemory(as: *Assembler) ParseError!Instruction.Memory { + const ptr_size: ?Instruction.Memory.PtrSize = blk: { + const pos = as.it.pos; + const ptr_size = as.parsePtrSize() catch |err| switch (err) { + error.UnexpectedToken => { + as.it.seekTo(pos); + break :blk null; + }, + else => return err, + }; + break :blk ptr_size; + }; + + try as.skip(1, .{.space}); + + // Supported rules and orderings. + const rules = .{ + .{ .open_br, .general_purpose, .close_br }, // [ general_purpose ] + .{ .open_br, .general_purpose, .plus, .disp, .close_br }, // [ general_purpose + disp ] + .{ .open_br, .general_purpose, .minus, .disp, .close_br }, // [ general_purpose - disp ] + .{ .open_br, .disp, .plus, .general_purpose, .close_br }, // [ disp + general_purpose ] + .{ .open_br, .general_purpose, .plus, .index, .close_br }, // [ general_purpose + index ] + .{ .open_br, .general_purpose, .plus, .index, .star, .scale, .close_br }, // [ general_purpose + index * scale ] + .{ .open_br, .index, .star, .scale, .plus, .general_purpose, .close_br }, // [ index * scale + general_purpose ] + .{ .open_br, .general_purpose, .plus, .index, .star, .scale, .plus, .disp, .close_br }, // [ general_purpose + index * scale + disp ] + .{ .open_br, .general_purpose, .plus, .index, .star, .scale, .minus, .disp, .close_br }, // [ general_purpose + index * scale - disp ] + .{ .open_br, .index, .star, .scale, .plus, .general_purpose, .plus, .disp, .close_br }, // [ index * scale + general_purpose + disp ] + .{ .open_br, .index, .star, .scale, .plus, .general_purpose, .minus, .disp, .close_br }, // [ index * scale + general_purpose - disp ] + .{ .open_br, .disp, .plus, .index, .star, .scale, .plus, .general_purpose, .close_br }, // [ disp + index * scale + general_purpose ] + .{ .open_br, .disp, .plus, .general_purpose, .plus, .index, .star, .scale, .close_br }, // [ disp + general_purpose + index * scale ] + .{ .open_br, .general_purpose, .plus, .disp, .plus, .index, .star, .scale, .close_br }, // [ general_purpose + disp + index * scale ] + .{ .open_br, .general_purpose, .minus, .disp, .plus, .index, .star, .scale, .close_br }, // [ general_purpose - disp + index * scale ] + .{ .open_br, .general_purpose, .plus, .disp, .plus, .scale, .star, .index, .close_br }, // [ general_purpose + disp + scale * index ] + .{ .open_br, .general_purpose, .minus, .disp, .plus, .scale, .star, .index, .close_br }, // [ general_purpose - disp + scale * index ] + .{ .open_br, .rip, .plus, .disp, .close_br }, // [ rip + disp ] + .{ .open_br, .rip, .minus, .disp, .close_br }, // [ rig - disp ] + .{ .segment, .colon, .disp }, // seg:disp + }; + + const pos = as.it.pos; + inline for (rules) |rule| { + if (as.parseMemoryRule(rule)) |res| { + if (res.rip) { + if (res.base != null or res.scale_index != null or res.offset != null) + return error.InvalidMemoryOperand; + return Instruction.Memory.initRip(ptr_size orelse .qword, res.disp orelse 0); + } + if (res.base) |base| { + if (res.rip) + return error.InvalidMemoryOperand; + if (res.offset) |offset| { + if (res.scale_index != null or res.disp != null) + return error.InvalidMemoryOperand; + return Instruction.Memory.initMoffs(base, offset); + } + return Instruction.Memory.initSib(ptr_size orelse .qword, .{ + .base = .{ .reg = base }, + .scale_index = res.scale_index, + .disp = res.disp orelse 0, + }); + } + return error.InvalidMemoryOperand; + } else |_| { + as.it.seekTo(pos); + } + } + + return error.InvalidOperand; + } + + const MemoryParseResult = struct { + rip: bool = false, + base: ?Register = null, + scale_index: ?Instruction.Memory.ScaleIndex = null, + disp: ?i32 = null, + offset: ?u64 = null, + }; + + fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!MemoryParseResult { + var res: MemoryParseResult = .{}; + inline for (rule, 0..) |cond, i| { + if (@typeInfo(@TypeOf(cond)) != .enum_literal) { + @compileError("unsupported condition type in the rule: " ++ @typeName(@TypeOf(cond))); + } + switch (cond) { + .open_br, .close_br, .plus, .minus, .star, .colon => { + _ = try as.expect(cond); + }, + .general_purpose, .segment => { + const tok = try as.expect(.string); + const base = registerFromString(as.source(tok)) orelse return error.InvalidMemoryOperand; + if (!base.isClass(cond)) return error.InvalidMemoryOperand; + res.base = base; + }, + .rip => { + const tok = try as.expect(.string); + if (!std.mem.eql(u8, as.source(tok), "rip")) return error.InvalidMemoryOperand; + res.rip = true; + }, + .index => { + const tok = try as.expect(.string); + const index = registerFromString(as.source(tok)) orelse + return error.InvalidMemoryOperand; + if (res.scale_index) |*si| { + si.index = index; + } else { + res.scale_index = .{ .scale = 1, .index = index }; + } + }, + .scale => { + const tok = try as.expect(.numeral); + const scale = try std.fmt.parseInt(u2, as.source(tok), 0); + if (res.scale_index) |*si| { + si.scale = scale; + } else { + res.scale_index = .{ .scale = scale, .index = undefined }; + } + }, + .disp => { + const tok = try as.expect(.numeral); + const is_neg = blk: { + if (i > 0) { + if (rule[i - 1] == .minus) break :blk true; + } + break :blk false; + }; + if (std.fmt.parseInt(i32, as.source(tok), 0)) |disp| { + res.disp = if (is_neg) -1 * disp else disp; + } else |err| switch (err) { + error.Overflow => { + if (is_neg) return err; + if (res.base) |base| { + if (!base.isClass(.segment)) return err; + } + const offset = try std.fmt.parseInt(u64, as.source(tok), 0); + res.offset = offset; + }, + else => return err, + } + }, + else => @compileError("unhandled operand output type: " ++ @tagName(cond)), + } + try as.skip(1, .{.space}); + } + return res; + } + + fn parsePtrSize(as: *Assembler) ParseError!Instruction.Memory.PtrSize { + const size = try as.expect(.string); + try as.skip(1, .{.space}); + const ptr = try as.expect(.string); + + const size_raw = as.source(size); + const ptr_raw = as.source(ptr); + const len = size_raw.len + ptr_raw.len + 1; + var buf: ["qword ptr".len]u8 = undefined; + if (len > buf.len) return error.InvalidPtrSize; + + for (size_raw, 0..) |c, i| { + buf[i] = std.ascii.toLower(c); + } + buf[size_raw.len] = ' '; + for (ptr_raw, 0..) |c, i| { + buf[size_raw.len + i + 1] = std.ascii.toLower(c); + } + + const slice = buf[0..len]; + if (std.mem.eql(u8, slice, "qword ptr")) return .qword; + if (std.mem.eql(u8, slice, "dword ptr")) return .dword; + if (std.mem.eql(u8, slice, "word ptr")) return .word; + if (std.mem.eql(u8, slice, "byte ptr")) return .byte; + if (std.mem.eql(u8, slice, "tbyte ptr")) return .tbyte; + return error.InvalidPtrSize; + } +}; + +test "assemble" { + const input = + \\int3 + \\mov rax, rbx + \\mov qword ptr [rbp], rax + \\mov qword ptr [rbp - 16], rax + \\mov qword ptr [16 + rbp], rax + \\mov rax, 0x10 + \\mov byte ptr [rbp - 0x10], 0x10 + \\mov word ptr [rbp + r12], r11w + \\mov word ptr [rbp + r12 * 2], r11w + \\mov word ptr [rbp + r12 * 2 - 16], r11w + \\mov dword ptr [rip - 16], r12d + \\mov rax, fs:0x0 + \\mov rax, gs:0x1000000000000000 + \\movzx r12, al + \\imul r12, qword ptr [rbp - 16], 6 + \\jmp 0x0 + \\jc 0x0 + \\jb 0x0 + \\sal rax, 1 + \\sal rax, 63 + \\shl rax, 63 + \\sar rax, 63 + \\shr rax, 63 + \\test byte ptr [rbp - 16], r12b + \\sal r12, cl + \\mul qword ptr [rip - 16] + \\div r12 + \\idiv byte ptr [rbp - 16] + \\cwde + \\cbw + \\cdqe + \\test byte ptr [rbp], ah + \\test byte ptr [r12], spl + \\cdq + \\cwd + \\cqo + \\test bl, 0x1 + \\mov rbx,0x8000000000000000 + \\movss xmm0, dword ptr [rbp] + \\movss xmm0, xmm1 + \\movss dword ptr [rbp - 16 + rax * 2], xmm7 + \\movss dword ptr [rbp - 16 + rax * 2], xmm8 + \\movss xmm15, xmm9 + \\movsd xmm8, qword ptr [rbp - 16] + \\movsd qword ptr [rbp - 8], xmm0 + \\movq xmm8, qword ptr [rbp - 16] + \\movq qword ptr [rbp - 16], xmm8 + \\ucomisd xmm0, qword ptr [rbp - 16] + \\fisttp qword ptr [rbp - 16] + \\fisttp word ptr [rip + 32] + \\fisttp dword ptr [rax] + \\fld tbyte ptr [rbp] + \\fld dword ptr [rbp] + \\xor bl, 0xff + \\ud2 + \\add rsp, -1 + \\add rsp, 0xff + \\mov sil, byte ptr [rax + rcx * 1] + \\ + ; + + // zig fmt: off + const expected = &[_]u8{ + 0xCC, + 0x48, 0x89, 0xD8, + 0x48, 0x89, 0x45, 0x00, + 0x48, 0x89, 0x45, 0xF0, + 0x48, 0x89, 0x45, 0x10, + 0x48, 0xC7, 0xC0, 0x10, 0x00, 0x00, 0x00, + 0xC6, 0x45, 0xF0, 0x10, + 0x66, 0x46, 0x89, 0x5C, 0x25, 0x00, + 0x66, 0x46, 0x89, 0x5C, 0x65, 0x00, + 0x66, 0x46, 0x89, 0x5C, 0x65, 0xF0, + 0x44, 0x89, 0x25, 0xF0, 0xFF, 0xFF, 0xFF, + 0x64, 0x48, 0x8B, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, + 0x65, 0x48, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x4C, 0x0F, 0xB6, 0xE0, + 0x4C, 0x6B, 0x65, 0xF0, 0x06, + 0xE9, 0x00, 0x00, 0x00, 0x00, + 0x0F, 0x82, 0x00, 0x00, 0x00, 0x00, + 0x0F, 0x82, 0x00, 0x00, 0x00, 0x00, + 0x48, 0xD1, 0xE0, + 0x48, 0xC1, 0xE0, 0x3F, + 0x48, 0xC1, 0xE0, 0x3F, + 0x48, 0xC1, 0xF8, 0x3F, + 0x48, 0xC1, 0xE8, 0x3F, + 0x44, 0x84, 0x65, 0xF0, + 0x49, 0xD3, 0xE4, + 0x48, 0xF7, 0x25, 0xF0, 0xFF, 0xFF, 0xFF, + 0x49, 0xF7, 0xF4, + 0xF6, 0x7D, 0xF0, + 0x98, + 0x66, 0x98, + 0x48, 0x98, + 0x84, 0x65, 0x00, + 0x41, 0x84, 0x24, 0x24, + 0x99, + 0x66, 0x99, + 0x48, 0x99, + 0xF6, 0xC3, 0x01, + 0x48, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0xF3, 0x0F, 0x10, 0x45, 0x00, + 0xF3, 0x0F, 0x10, 0xC1, + 0xF3, 0x0F, 0x11, 0x7C, 0x45, 0xF0, + 0xF3, 0x44, 0x0F, 0x11, 0x44, 0x45, 0xF0, + 0xF3, 0x45, 0x0F, 0x10, 0xF9, + 0xF2, 0x44, 0x0F, 0x10, 0x45, 0xF0, + 0xF2, 0x0F, 0x11, 0x45, 0xF8, + 0x66, 0x4C, 0x0F, 0x6E, 0x45, 0xF0, + 0x66, 0x4C, 0x0F, 0x7E, 0x45, 0xF0, + 0x66, 0x0F, 0x2E, 0x45, 0xF0, + 0xDD, 0x4D, 0xF0, + 0xDF, 0x0D, 0x20, 0x00, 0x00, 0x00, + 0xDB, 0x08, + 0xDB, 0x6D, 0x00, + 0xD9, 0x45, 0x00, + 0x80, 0xF3, 0xFF, + 0x0F, 0x0B, + 0x48, 0x83, 0xC4, 0xFF, + 0x48, 0x81, 0xC4, 0xFF, 0x00, 0x00, 0x00, + 0x40, 0x8A, 0x34, 0x08, + }; + // zig fmt: on + + var as = Assembler.init(input); + var output = std.array_list.Managed(u8).init(testing.allocator); + defer output.deinit(); + try as.assemble(output.writer()); + try expectEqualHexStrings(expected, output.items, input); +} + +test "assemble - Jcc" { + const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{ + .{ .ja, 0x87 }, + .{ .jae, 0x83 }, + .{ .jb, 0x82 }, + .{ .jbe, 0x86 }, + .{ .jc, 0x82 }, + .{ .je, 0x84 }, + .{ .jg, 0x8f }, + .{ .jge, 0x8d }, + .{ .jl, 0x8c }, + .{ .jle, 0x8e }, + .{ .jna, 0x86 }, + .{ .jnae, 0x82 }, + .{ .jnb, 0x83 }, + .{ .jnbe, 0x87 }, + .{ .jnc, 0x83 }, + .{ .jne, 0x85 }, + .{ .jng, 0x8e }, + .{ .jnge, 0x8c }, + .{ .jnl, 0x8d }, + .{ .jnle, 0x8f }, + .{ .jno, 0x81 }, + .{ .jnp, 0x8b }, + .{ .jns, 0x89 }, + .{ .jnz, 0x85 }, + .{ .jo, 0x80 }, + .{ .jp, 0x8a }, + .{ .jpe, 0x8a }, + .{ .jpo, 0x8b }, + .{ .js, 0x88 }, + .{ .jz, 0x84 }, + }; + + inline for (&mnemonics) |mnemonic| { + const input = @tagName(mnemonic[0]) ++ " 0x0"; + const expected = [_]u8{ 0x0f, mnemonic[1], 0x0, 0x0, 0x0, 0x0 }; + var as = Assembler.init(input); + var output = std.array_list.Managed(u8).init(testing.allocator); + defer output.deinit(); + try as.assemble(output.writer()); + try expectEqualHexStrings(&expected, output.items, input); + } +} + +test "assemble - SETcc" { + const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{ + .{ .seta, 0x97 }, + .{ .setae, 0x93 }, + .{ .setb, 0x92 }, + .{ .setbe, 0x96 }, + .{ .setc, 0x92 }, + .{ .sete, 0x94 }, + .{ .setg, 0x9f }, + .{ .setge, 0x9d }, + .{ .setl, 0x9c }, + .{ .setle, 0x9e }, + .{ .setna, 0x96 }, + .{ .setnae, 0x92 }, + .{ .setnb, 0x93 }, + .{ .setnbe, 0x97 }, + .{ .setnc, 0x93 }, + .{ .setne, 0x95 }, + .{ .setng, 0x9e }, + .{ .setnge, 0x9c }, + .{ .setnl, 0x9d }, + .{ .setnle, 0x9f }, + .{ .setno, 0x91 }, + .{ .setnp, 0x9b }, + .{ .setns, 0x99 }, + .{ .setnz, 0x95 }, + .{ .seto, 0x90 }, + .{ .setp, 0x9a }, + .{ .setpe, 0x9a }, + .{ .setpo, 0x9b }, + .{ .sets, 0x98 }, + .{ .setz, 0x94 }, + }; + + inline for (&mnemonics) |mnemonic| { + const input = @tagName(mnemonic[0]) ++ " al"; + const expected = [_]u8{ 0x0f, mnemonic[1], 0xC0 }; + var as = Assembler.init(input); + var output = std.array_list.Managed(u8).init(testing.allocator); + defer output.deinit(); + try as.assemble(output.writer()); + try expectEqualHexStrings(&expected, output.items, input); + } +} + +test "assemble - CMOVcc" { + const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{ + .{ .cmova, 0x47 }, + .{ .cmovae, 0x43 }, + .{ .cmovb, 0x42 }, + .{ .cmovbe, 0x46 }, + .{ .cmovc, 0x42 }, + .{ .cmove, 0x44 }, + .{ .cmovg, 0x4f }, + .{ .cmovge, 0x4d }, + .{ .cmovl, 0x4c }, + .{ .cmovle, 0x4e }, + .{ .cmovna, 0x46 }, + .{ .cmovnae, 0x42 }, + .{ .cmovnb, 0x43 }, + .{ .cmovnbe, 0x47 }, + .{ .cmovnc, 0x43 }, + .{ .cmovne, 0x45 }, + .{ .cmovng, 0x4e }, + .{ .cmovnge, 0x4c }, + .{ .cmovnl, 0x4d }, + .{ .cmovnle, 0x4f }, + .{ .cmovno, 0x41 }, + .{ .cmovnp, 0x4b }, + .{ .cmovns, 0x49 }, + .{ .cmovnz, 0x45 }, + .{ .cmovo, 0x40 }, + .{ .cmovp, 0x4a }, + .{ .cmovpe, 0x4a }, + .{ .cmovpo, 0x4b }, + .{ .cmovs, 0x48 }, + .{ .cmovz, 0x44 }, + }; + + inline for (&mnemonics) |mnemonic| { + const input = @tagName(mnemonic[0]) ++ " rax, rbx"; + const expected = [_]u8{ 0x48, 0x0f, mnemonic[1], 0xC3 }; + var as = Assembler.init(input); + var output = std.array_list.Managed(u8).init(testing.allocator); + defer output.deinit(); + try as.assemble(output.writer()); + try expectEqualHexStrings(&expected, output.items, input); + } +} diff --git a/src/codegen/x86_64/encodings.zon b/src/codegen/x86_64/encodings.zon new file mode 100644 index 0000000000000000000000000000000000000000..ea136569ce939d97982931afef4c5ae54a07322a --- /dev/null +++ b/src/codegen/x86_64/encodings.zon @@ -0,0 +1,2700 @@ +// zig fmt: off +.{ + // General-purpose + .{ .aaa, .z, .{}, .{ 0x37 }, 0, .none, .@"32bit" }, + + .{ .aad, .z, .{ }, .{ 0xd5, 0x0a }, 0, .none, .@"32bit" }, + .{ .aad, .zi, .{ .imm8 }, .{ 0xd5 }, 0, .none, .@"32bit" }, + + .{ .aam, .z, .{ }, .{ 0xd4, 0x0a }, 0, .none, .@"32bit" }, + .{ .aam, .zi, .{ .imm8 }, .{ 0xd4 }, 0, .none, .@"32bit" }, + + .{ .aas, .z, .{}, .{ 0x3f }, 0, .none, .@"32bit" }, + + .{ .adc, .zi, .{ .al, .imm8 }, .{ 0x14 }, 0, .none, .none }, + .{ .adc, .zi, .{ .ax, .imm16 }, .{ 0x15 }, 0, .short, .none }, + .{ .adc, .zi, .{ .eax, .imm32 }, .{ 0x15 }, 0, .none, .none }, + .{ .adc, .zi, .{ .rax, .imm32s }, .{ 0x15 }, 0, .long, .none }, + .{ .adc, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 2, .none, .none }, + .{ .adc, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 2, .rex, .none }, + .{ .adc, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 2, .short, .none }, + .{ .adc, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 2, .none, .none }, + .{ .adc, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 2, .long, .none }, + .{ .adc, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 2, .short, .none }, + .{ .adc, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 2, .none, .none }, + .{ .adc, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 2, .long, .none }, + .{ .adc, .mr, .{ .rm8, .r8 }, .{ 0x10 }, 0, .none, .none }, + .{ .adc, .mr, .{ .rm8, .r8 }, .{ 0x10 }, 0, .rex, .none }, + .{ .adc, .mr, .{ .rm16, .r16 }, .{ 0x11 }, 0, .short, .none }, + .{ .adc, .mr, .{ .rm32, .r32 }, .{ 0x11 }, 0, .none, .none }, + .{ .adc, .mr, .{ .rm64, .r64 }, .{ 0x11 }, 0, .long, .none }, + .{ .adc, .rm, .{ .r8, .rm8 }, .{ 0x12 }, 0, .none, .none }, + .{ .adc, .rm, .{ .r8, .rm8 }, .{ 0x12 }, 0, .rex, .none }, + .{ .adc, .rm, .{ .r16, .rm16 }, .{ 0x13 }, 0, .short, .none }, + .{ .adc, .rm, .{ .r32, .rm32 }, .{ 0x13 }, 0, .none, .none }, + .{ .adc, .rm, .{ .r64, .rm64 }, .{ 0x13 }, 0, .long, .none }, + + .{ .add, .zi, .{ .al, .imm8 }, .{ 0x04 }, 0, .none, .none }, + .{ .add, .zi, .{ .ax, .imm16 }, .{ 0x05 }, 0, .short, .none }, + .{ .add, .zi, .{ .eax, .imm32 }, .{ 0x05 }, 0, .none, .none }, + .{ .add, .zi, .{ .rax, .imm32s }, .{ 0x05 }, 0, .long, .none }, + .{ .add, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 0, .none, .none }, + .{ .add, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 0, .rex, .none }, + .{ .add, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 0, .short, .none }, + .{ .add, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 0, .none, .none }, + .{ .add, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 0, .long, .none }, + .{ .add, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 0, .short, .none }, + .{ .add, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 0, .none, .none }, + .{ .add, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 0, .long, .none }, + .{ .add, .mr, .{ .rm8, .r8 }, .{ 0x00 }, 0, .none, .none }, + .{ .add, .mr, .{ .rm8, .r8 }, .{ 0x00 }, 0, .rex, .none }, + .{ .add, .mr, .{ .rm16, .r16 }, .{ 0x01 }, 0, .short, .none }, + .{ .add, .mr, .{ .rm32, .r32 }, .{ 0x01 }, 0, .none, .none }, + .{ .add, .mr, .{ .rm64, .r64 }, .{ 0x01 }, 0, .long, .none }, + .{ .add, .rm, .{ .r8, .rm8 }, .{ 0x02 }, 0, .none, .none }, + .{ .add, .rm, .{ .r8, .rm8 }, .{ 0x02 }, 0, .rex, .none }, + .{ .add, .rm, .{ .r16, .rm16 }, .{ 0x03 }, 0, .short, .none }, + .{ .add, .rm, .{ .r32, .rm32 }, .{ 0x03 }, 0, .none, .none }, + .{ .add, .rm, .{ .r64, .rm64 }, .{ 0x03 }, 0, .long, .none }, + + .{ .@"and", .zi, .{ .al, .imm8 }, .{ 0x24 }, 0, .none, .none }, + .{ .@"and", .zi, .{ .ax, .imm16 }, .{ 0x25 }, 0, .short, .none }, + .{ .@"and", .zi, .{ .eax, .imm32 }, .{ 0x25 }, 0, .none, .none }, + .{ .@"and", .zi, .{ .rax, .imm32s }, .{ 0x25 }, 0, .long, .none }, + .{ .@"and", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 4, .none, .none }, + .{ .@"and", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 4, .rex, .none }, + .{ .@"and", .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 4, .short, .none }, + .{ .@"and", .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 4, .none, .none }, + .{ .@"and", .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 4, .long, .none }, + .{ .@"and", .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 4, .short, .none }, + .{ .@"and", .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 4, .none, .none }, + .{ .@"and", .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 4, .long, .none }, + .{ .@"and", .mr, .{ .rm8, .r8 }, .{ 0x20 }, 0, .none, .none }, + .{ .@"and", .mr, .{ .rm8, .r8 }, .{ 0x20 }, 0, .rex, .none }, + .{ .@"and", .mr, .{ .rm16, .r16 }, .{ 0x21 }, 0, .short, .none }, + .{ .@"and", .mr, .{ .rm32, .r32 }, .{ 0x21 }, 0, .none, .none }, + .{ .@"and", .mr, .{ .rm64, .r64 }, .{ 0x21 }, 0, .long, .none }, + .{ .@"and", .rm, .{ .r8, .rm8 }, .{ 0x22 }, 0, .none, .none }, + .{ .@"and", .rm, .{ .r8, .rm8 }, .{ 0x22 }, 0, .rex, .none }, + .{ .@"and", .rm, .{ .r16, .rm16 }, .{ 0x23 }, 0, .short, .none }, + .{ .@"and", .rm, .{ .r32, .rm32 }, .{ 0x23 }, 0, .none, .none }, + .{ .@"and", .rm, .{ .r64, .rm64 }, .{ 0x23 }, 0, .long, .none }, + + .{ .arpl, .mr, .{ .rm16, .r16 }, .{ 0x63 }, 0, .none, .@"32bit" }, + + .{ .bound, .rm, .{ .r16, .m }, .{ 0x62 }, 0, .short, .@"32bit" }, + .{ .bound, .rm, .{ .r32, .m }, .{ 0x62 }, 0, .short, .@"32bit" }, + + .{ .bsf, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0xbc }, 0, .short, .none }, + .{ .bsf, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xbc }, 0, .none, .none }, + .{ .bsf, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0xbc }, 0, .long, .none }, + + .{ .bsr, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0xbd }, 0, .short, .none }, + .{ .bsr, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xbd }, 0, .none, .none }, + .{ .bsr, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0xbd }, 0, .long, .none }, + + .{ .bswap, .o, .{ .r32 }, .{ 0x0f, 0xc8 }, 0, .none, .none }, + .{ .bswap, .o, .{ .r64 }, .{ 0x0f, 0xc8 }, 0, .long, .none }, + + .{ .bt, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xa3 }, 0, .short, .none }, + .{ .bt, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xa3 }, 0, .none, .none }, + .{ .bt, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xa3 }, 0, .long, .none }, + .{ .bt, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 4, .short, .none }, + .{ .bt, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 4, .none, .none }, + .{ .bt, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 4, .long, .none }, + + .{ .btc, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xbb }, 0, .short, .none }, + .{ .btc, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xbb }, 0, .none, .none }, + .{ .btc, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xbb }, 0, .long, .none }, + .{ .btc, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 7, .short, .none }, + .{ .btc, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 7, .none, .none }, + .{ .btc, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 7, .long, .none }, + + .{ .btr, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xb3 }, 0, .short, .none }, + .{ .btr, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xb3 }, 0, .none, .none }, + .{ .btr, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xb3 }, 0, .long, .none }, + .{ .btr, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 6, .short, .none }, + .{ .btr, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 6, .none, .none }, + .{ .btr, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 6, .long, .none }, + + .{ .bts, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xab }, 0, .short, .none }, + .{ .bts, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xab }, 0, .none, .none }, + .{ .bts, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xab }, 0, .long, .none }, + .{ .bts, .mi, .{ .rm16, .imm8 }, .{ 0x0f, 0xba }, 5, .short, .none }, + .{ .bts, .mi, .{ .rm32, .imm8 }, .{ 0x0f, 0xba }, 5, .none, .none }, + .{ .bts, .mi, .{ .rm64, .imm8 }, .{ 0x0f, 0xba }, 5, .long, .none }, + + .{ .call, .d, .{ .rel32 }, .{ 0xe8 }, 0, .none, .none }, + .{ .call, .m, .{ .rm32 }, .{ 0xff }, 2, .none, .@"32bit" }, + .{ .call, .m, .{ .rm64 }, .{ 0xff }, 2, .none, .@"64bit" }, + + .{ .cbw, .z, .{}, .{ 0x98 }, 0, .short, .none }, + .{ .cwde, .z, .{}, .{ 0x98 }, 0, .none, .none }, + .{ .cdqe, .z, .{}, .{ 0x98 }, 0, .long, .none }, + + .{ .clac, .z, .{}, .{ 0x0f, 0x01, 0xca }, 0, .none, .smap }, + + .{ .clc, .z, .{}, .{ 0xf8 }, 0, .none, .none }, + + .{ .cld, .z, .{}, .{ 0xfc }, 0, .none, .none }, + + .{ .cldemote, .m, .{ .m8 }, .{ 0x0f, 0x1c }, 0, .none, .cldemote }, + + .{ .clflush, .m, .{ .m8 }, .{ 0x0f, 0xae }, 7, .none, .none }, + + .{ .clflushopt, .m, .{ .m8 }, .{ 0x66, 0x0f, 0xae }, 7, .none, .clflushopt }, + + .{ .cli, .z, .{}, .{ 0xfa }, 0, .none, .none }, + + .{ .clrssbsy, .m, .{ .m64 }, .{ 0xf3, 0x0f, 0xae }, 6, .none, .shstk }, + + .{ .clts, .z, .{}, .{ 0x0f, 0x06 }, 0, .none, .none }, + + .{ .clui, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr }, + + .{ .clwb, .m, .{ .m8 }, .{ 0x66, 0x0f, 0xae }, 6, .none, .clwb }, + + .{ .cmc, .z, .{}, .{ 0xf5 }, 0, .none, .none }, + + .{ .cmova, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x47 }, 0, .short, .cmov }, + .{ .cmova, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x47 }, 0, .none, .cmov }, + .{ .cmova, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x47 }, 0, .long, .cmov }, + .{ .cmovae, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x43 }, 0, .short, .cmov }, + .{ .cmovae, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x43 }, 0, .none, .cmov }, + .{ .cmovae, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x43 }, 0, .long, .cmov }, + .{ .cmovb, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x42 }, 0, .short, .cmov }, + .{ .cmovb, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x42 }, 0, .none, .cmov }, + .{ .cmovb, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x42 }, 0, .long, .cmov }, + .{ .cmovbe, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x46 }, 0, .short, .cmov }, + .{ .cmovbe, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x46 }, 0, .none, .cmov }, + .{ .cmovbe, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x46 }, 0, .long, .cmov }, + .{ .cmovc, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x42 }, 0, .short, .cmov }, + .{ .cmovc, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x42 }, 0, .none, .cmov }, + .{ .cmovc, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x42 }, 0, .long, .cmov }, + .{ .cmove, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x44 }, 0, .short, .cmov }, + .{ .cmove, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x44 }, 0, .none, .cmov }, + .{ .cmove, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x44 }, 0, .long, .cmov }, + .{ .cmovg, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4f }, 0, .short, .cmov }, + .{ .cmovg, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4f }, 0, .none, .cmov }, + .{ .cmovg, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4f }, 0, .long, .cmov }, + .{ .cmovge, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4d }, 0, .short, .cmov }, + .{ .cmovge, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4d }, 0, .none, .cmov }, + .{ .cmovge, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4d }, 0, .long, .cmov }, + .{ .cmovl, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4c }, 0, .short, .cmov }, + .{ .cmovl, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4c }, 0, .none, .cmov }, + .{ .cmovl, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4c }, 0, .long, .cmov }, + .{ .cmovle, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4e }, 0, .short, .cmov }, + .{ .cmovle, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4e }, 0, .none, .cmov }, + .{ .cmovle, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4e }, 0, .long, .cmov }, + .{ .cmovna, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x46 }, 0, .short, .cmov }, + .{ .cmovna, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x46 }, 0, .none, .cmov }, + .{ .cmovna, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x46 }, 0, .long, .cmov }, + .{ .cmovnae, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x42 }, 0, .short, .cmov }, + .{ .cmovnae, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x42 }, 0, .none, .cmov }, + .{ .cmovnae, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x42 }, 0, .long, .cmov }, + .{ .cmovnb, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x43 }, 0, .short, .cmov }, + .{ .cmovnb, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x43 }, 0, .none, .cmov }, + .{ .cmovnb, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x43 }, 0, .long, .cmov }, + .{ .cmovnbe, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x47 }, 0, .short, .cmov }, + .{ .cmovnbe, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x47 }, 0, .none, .cmov }, + .{ .cmovnbe, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x47 }, 0, .long, .cmov }, + .{ .cmovnc, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x43 }, 0, .short, .cmov }, + .{ .cmovnc, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x43 }, 0, .none, .cmov }, + .{ .cmovnc, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x43 }, 0, .long, .cmov }, + .{ .cmovne, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x45 }, 0, .short, .cmov }, + .{ .cmovne, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x45 }, 0, .none, .cmov }, + .{ .cmovne, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x45 }, 0, .long, .cmov }, + .{ .cmovng, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4e }, 0, .short, .cmov }, + .{ .cmovng, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4e }, 0, .none, .cmov }, + .{ .cmovng, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4e }, 0, .long, .cmov }, + .{ .cmovnge, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4c }, 0, .short, .cmov }, + .{ .cmovnge, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4c }, 0, .none, .cmov }, + .{ .cmovnge, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4c }, 0, .long, .cmov }, + .{ .cmovnl, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4d }, 0, .short, .cmov }, + .{ .cmovnl, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4d }, 0, .none, .cmov }, + .{ .cmovnl, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4d }, 0, .long, .cmov }, + .{ .cmovnle, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4f }, 0, .short, .cmov }, + .{ .cmovnle, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4f }, 0, .none, .cmov }, + .{ .cmovnle, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4f }, 0, .long, .cmov }, + .{ .cmovno, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x41 }, 0, .short, .cmov }, + .{ .cmovno, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x41 }, 0, .none, .cmov }, + .{ .cmovno, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x41 }, 0, .long, .cmov }, + .{ .cmovnp, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4b }, 0, .short, .cmov }, + .{ .cmovnp, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4b }, 0, .none, .cmov }, + .{ .cmovnp, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4b }, 0, .long, .cmov }, + .{ .cmovns, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x49 }, 0, .short, .cmov }, + .{ .cmovns, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x49 }, 0, .none, .cmov }, + .{ .cmovns, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x49 }, 0, .long, .cmov }, + .{ .cmovnz, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x45 }, 0, .short, .cmov }, + .{ .cmovnz, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x45 }, 0, .none, .cmov }, + .{ .cmovnz, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x45 }, 0, .long, .cmov }, + .{ .cmovo, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x40 }, 0, .short, .cmov }, + .{ .cmovo, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x40 }, 0, .none, .cmov }, + .{ .cmovo, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x40 }, 0, .long, .cmov }, + .{ .cmovp, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4a }, 0, .short, .cmov }, + .{ .cmovp, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4a }, 0, .none, .cmov }, + .{ .cmovp, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4a }, 0, .long, .cmov }, + .{ .cmovpe, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4a }, 0, .short, .cmov }, + .{ .cmovpe, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4a }, 0, .none, .cmov }, + .{ .cmovpe, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4a }, 0, .long, .cmov }, + .{ .cmovpo, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x4b }, 0, .short, .cmov }, + .{ .cmovpo, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x4b }, 0, .none, .cmov }, + .{ .cmovpo, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x4b }, 0, .long, .cmov }, + .{ .cmovs, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x48 }, 0, .short, .cmov }, + .{ .cmovs, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x48 }, 0, .none, .cmov }, + .{ .cmovs, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x48 }, 0, .long, .cmov }, + .{ .cmovz, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x44 }, 0, .short, .cmov }, + .{ .cmovz, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0x44 }, 0, .none, .cmov }, + .{ .cmovz, .rm, .{ .r64, .rm64 }, .{ 0x0f, 0x44 }, 0, .long, .cmov }, + + .{ .cmp, .zi, .{ .al, .imm8 }, .{ 0x3c }, 0, .none, .none }, + .{ .cmp, .zi, .{ .ax, .imm16 }, .{ 0x3d }, 0, .short, .none }, + .{ .cmp, .zi, .{ .eax, .imm32 }, .{ 0x3d }, 0, .none, .none }, + .{ .cmp, .zi, .{ .rax, .imm32s }, .{ 0x3d }, 0, .long, .none }, + .{ .cmp, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 7, .none, .none }, + .{ .cmp, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 7, .rex, .none }, + .{ .cmp, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 7, .short, .none }, + .{ .cmp, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 7, .none, .none }, + .{ .cmp, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 7, .long, .none }, + .{ .cmp, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 7, .short, .none }, + .{ .cmp, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 7, .none, .none }, + .{ .cmp, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 7, .long, .none }, + .{ .cmp, .mr, .{ .rm8, .r8 }, .{ 0x38 }, 0, .none, .none }, + .{ .cmp, .mr, .{ .rm8, .r8 }, .{ 0x38 }, 0, .rex, .none }, + .{ .cmp, .mr, .{ .rm16, .r16 }, .{ 0x39 }, 0, .short, .none }, + .{ .cmp, .mr, .{ .rm32, .r32 }, .{ 0x39 }, 0, .none, .none }, + .{ .cmp, .mr, .{ .rm64, .r64 }, .{ 0x39 }, 0, .long, .none }, + .{ .cmp, .rm, .{ .r8, .rm8 }, .{ 0x3a }, 0, .none, .none }, + .{ .cmp, .rm, .{ .r8, .rm8 }, .{ 0x3a }, 0, .rex, .none }, + .{ .cmp, .rm, .{ .r16, .rm16 }, .{ 0x3b }, 0, .short, .none }, + .{ .cmp, .rm, .{ .r32, .rm32 }, .{ 0x3b }, 0, .none, .none }, + .{ .cmp, .rm, .{ .r64, .rm64 }, .{ 0x3b }, 0, .long, .none }, + + .{ .cmps, .z, .{ .m8, .m8 }, .{ 0xa6 }, 0, .none, .none }, + .{ .cmps, .z, .{ .m16, .m16 }, .{ 0xa7 }, 0, .short, .none }, + .{ .cmps, .z, .{ .m32, .m32 }, .{ 0xa7 }, 0, .none, .none }, + .{ .cmps, .z, .{ .m64, .m64 }, .{ 0xa7 }, 0, .long, .none }, + .{ .cmpsb, .z, .{ }, .{ 0xa6 }, 0, .none, .none }, + .{ .cmpsw, .z, .{ }, .{ 0xa7 }, 0, .short, .none }, + .{ .cmpsd, .z, .{ }, .{ 0xa7 }, 0, .none, .none }, + .{ .cmpsq, .z, .{ }, .{ 0xa7 }, 0, .long, .none }, + + .{ .cmpxchg, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xb0 }, 0, .none, .none }, + .{ .cmpxchg, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xb0 }, 0, .rex, .none }, + .{ .cmpxchg, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xb1 }, 0, .short, .none }, + .{ .cmpxchg, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xb1 }, 0, .none, .none }, + .{ .cmpxchg, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xb1 }, 0, .long, .none }, + + .{ .cmpxchg8b, .m, .{ .m64 }, .{ 0x0f, 0xc7 }, 1, .none, .none }, + .{ .cmpxchg16b, .m, .{ .m128 }, .{ 0x0f, 0xc7 }, 1, .long, .none }, + + .{ .cpuid, .z, .{}, .{ 0x0f, 0xa2 }, 0, .none, .none }, + + .{ .cwd, .z, .{}, .{ 0x99 }, 0, .short, .none }, + .{ .cdq, .z, .{}, .{ 0x99 }, 0, .none, .none }, + .{ .cqo, .z, .{}, .{ 0x99 }, 0, .long, .none }, + + .{ .daa, .z, .{}, .{ 0x27 }, 0, .none, .@"32bit" }, + + .{ .das, .z, .{}, .{ 0x27 }, 0, .none, .@"32bit" }, + + .{ .dec, .m, .{ .rm8 }, .{ 0xfe }, 1, .none, .none }, + .{ .dec, .m, .{ .rm8 }, .{ 0xfe }, 1, .rex, .none }, + .{ .dec, .m, .{ .rm16 }, .{ 0xff }, 1, .short, .none }, + .{ .dec, .m, .{ .rm32 }, .{ 0xff }, 1, .none, .none }, + .{ .dec, .m, .{ .rm64 }, .{ 0xff }, 1, .long, .none }, + + .{ .div, .m, .{ .rm8 }, .{ 0xf6 }, 6, .none, .none }, + .{ .div, .m, .{ .rm8 }, .{ 0xf6 }, 6, .rex, .none }, + .{ .div, .m, .{ .rm16 }, .{ 0xf7 }, 6, .short, .none }, + .{ .div, .m, .{ .rm32 }, .{ 0xf7 }, 6, .none, .none }, + .{ .div, .m, .{ .rm64 }, .{ 0xf7 }, 6, .long, .none }, + + .{ .endbr32, .z, .{}, .{ 0xf3, 0x0f, 0x1e, 0xfb }, 0, .none, .none }, + + .{ .endbr64, .z, .{}, .{ 0xf3, 0x0f, 0x1e, 0xfa }, 0, .none, .none }, + + .{ .enqcmd, .rm, .{ .r32, .m }, .{ 0xf2, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, + .{ .enqcmd, .rm, .{ .r64, .m }, .{ 0xf2, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, + + .{ .enqcmds, .rm, .{ .r32, .m }, .{ 0xf3, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, + .{ .enqcmds, .rm, .{ .r64, .m }, .{ 0xf3, 0x0f, 0x38, 0xf8 }, 0, .none, .enqcmd }, + + .{ .enter, .ii, .{ .imm16, .imm8 }, .{ 0xc8 }, 0, .none, .none }, + + .{ .hlt, .z, .{}, .{ 0xf4 }, 0, .none, .none }, + + .{ .hreset, .ia, .{ .imm8 }, .{ 0xf3, 0x0f, 0x3a, 0xf0 }, 0, .none, .hreset }, + .{ .hreset, .ia, .{ .imm8, .eax }, .{ 0xf3, 0x0f, 0x3a, 0xf0 }, 0, .none, .hreset }, + + .{ .idiv, .m, .{ .rm8 }, .{ 0xf6 }, 7, .none, .none }, + .{ .idiv, .m, .{ .rm8 }, .{ 0xf6 }, 7, .rex, .none }, + .{ .idiv, .m, .{ .rm16 }, .{ 0xf7 }, 7, .short, .none }, + .{ .idiv, .m, .{ .rm32 }, .{ 0xf7 }, 7, .none, .none }, + .{ .idiv, .m, .{ .rm64 }, .{ 0xf7 }, 7, .long, .none }, + + .{ .imul, .m, .{ .rm8 }, .{ 0xf6 }, 5, .none, .none }, + .{ .imul, .m, .{ .rm8 }, .{ 0xf6 }, 5, .rex, .none }, + .{ .imul, .m, .{ .rm16, }, .{ 0xf7 }, 5, .short, .none }, + .{ .imul, .m, .{ .rm32, }, .{ 0xf7 }, 5, .none, .none }, + .{ .imul, .m, .{ .rm64, }, .{ 0xf7 }, 5, .long, .none }, + .{ .imul, .rm, .{ .r16, .rm16, }, .{ 0x0f, 0xaf }, 0, .short, .none }, + .{ .imul, .rm, .{ .r32, .rm32, }, .{ 0x0f, 0xaf }, 0, .none, .none }, + .{ .imul, .rm, .{ .r64, .rm64, }, .{ 0x0f, 0xaf }, 0, .long, .none }, + .{ .imul, .rmi, .{ .r16, .rm16, .imm8s }, .{ 0x6b }, 0, .short, .none }, + .{ .imul, .rmi, .{ .r32, .rm32, .imm8s }, .{ 0x6b }, 0, .none, .none }, + .{ .imul, .rmi, .{ .r64, .rm64, .imm8s }, .{ 0x6b }, 0, .long, .none }, + .{ .imul, .rmi, .{ .r16, .rm16, .imm16 }, .{ 0x69 }, 0, .short, .none }, + .{ .imul, .rmi, .{ .r32, .rm32, .imm32 }, .{ 0x69 }, 0, .none, .none }, + .{ .imul, .rmi, .{ .r64, .rm64, .imm32 }, .{ 0x69 }, 0, .long, .none }, + + .{ .in, .zi, .{ .al, .imm8 }, .{ 0xe4 }, 0, .none, .none }, + .{ .in, .zi, .{ .ax, .imm8 }, .{ 0xe5 }, 0, .short, .none }, + .{ .in, .zi, .{ .eax, .imm8 }, .{ 0xe5 }, 0, .none, .none }, + .{ .in, .z, .{ .al, .dx }, .{ 0xec }, 0, .none, .none }, + .{ .in, .z, .{ .ax, .dx }, .{ 0xed }, 0, .short, .none }, + .{ .in, .z, .{ .eax, .dx }, .{ 0xed }, 0, .none, .none }, + + .{ .inc, .m, .{ .rm8 }, .{ 0xfe }, 0, .none, .none }, + .{ .inc, .m, .{ .rm8 }, .{ 0xfe }, 0, .rex, .none }, + .{ .inc, .m, .{ .rm16 }, .{ 0xff }, 0, .short, .none }, + .{ .inc, .m, .{ .rm32 }, .{ 0xff }, 0, .none, .none }, + .{ .inc, .m, .{ .rm64 }, .{ 0xff }, 0, .long, .none }, + + .{ .incsspd, .m, .{ .r32 }, .{ 0xf3, 0x0f, 0xae }, 5, .none, .shstk }, + .{ .incsspq, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xae }, 5, .long, .shstk }, + + .{ .ins, .z, .{ .m8, .dx }, .{ 0x6c }, 0, .none, .none }, + .{ .ins, .z, .{ .m16, .dx }, .{ 0x6d }, 0, .short, .none }, + .{ .ins, .z, .{ .m32, .dx }, .{ 0x6d }, 0, .none, .none }, + .{ .insb, .z, .{ }, .{ 0x6c }, 0, .none, .none }, + .{ .insw, .z, .{ }, .{ 0x6d }, 0, .short, .none }, + .{ .insd, .z, .{ }, .{ 0x6d }, 0, .none, .none }, + + .{ .int3, .z, .{ }, .{ 0xcc }, 0, .none, .none }, + .{ .int, .i, .{ .imm8 }, .{ 0xcd }, 0, .none, .none }, + .{ .into, .z, .{ }, .{ 0xce }, 0, .none, .@"32bit" }, + .{ .int1, .z, .{ }, .{ 0xf1 }, 0, .none, .none }, + + .{ .invd, .z, .{}, .{ 0x0f, 0x08 }, 0, .none, .none }, + + .{ .invlpg, .m, .{ .m }, .{ 0x0f, 0x01 }, 7, .none, .none }, + + .{ .invpcid, .rm, .{ .r32, .m128 }, .{ 0x66, 0x0f, 0x38, 0x82 }, 0, .none, .@"invpcid 32bit" }, + .{ .invpcid, .rm, .{ .r64, .m128 }, .{ 0x66, 0x0f, 0x38, 0x82 }, 0, .none, .@"invpcid 64bit" }, + + .{ .iretw, .z, .{}, .{ 0xcf }, 0, .short, .none }, + .{ .iretd, .z, .{}, .{ 0xcf }, 0, .none, .none }, + .{ .iret, .z, .{}, .{ 0xcf }, 0, .none, .none }, + .{ .iretq, .z, .{}, .{ 0xcf }, 0, .long, .none }, + + .{ .ja, .d, .{ .rel32 }, .{ 0x0f, 0x87 }, 0, .none, .none }, + .{ .jae, .d, .{ .rel32 }, .{ 0x0f, 0x83 }, 0, .none, .none }, + .{ .jb, .d, .{ .rel32 }, .{ 0x0f, 0x82 }, 0, .none, .none }, + .{ .jbe, .d, .{ .rel32 }, .{ 0x0f, 0x86 }, 0, .none, .none }, + .{ .jc, .d, .{ .rel32 }, .{ 0x0f, 0x82 }, 0, .none, .none }, + .{ .jcxz, .d, .{ .rel8 }, .{ 0xe3 }, 0, .short, .@"32bit" }, + .{ .jecxz, .d, .{ .rel8 }, .{ 0xe3 }, 0, .none, .@"32bit" }, + .{ .jrcxz, .d, .{ .rel8 }, .{ 0xe3 }, 0, .none, .@"64bit" }, + .{ .je, .d, .{ .rel32 }, .{ 0x0f, 0x84 }, 0, .none, .none }, + .{ .jg, .d, .{ .rel32 }, .{ 0x0f, 0x8f }, 0, .none, .none }, + .{ .jge, .d, .{ .rel32 }, .{ 0x0f, 0x8d }, 0, .none, .none }, + .{ .jl, .d, .{ .rel32 }, .{ 0x0f, 0x8c }, 0, .none, .none }, + .{ .jle, .d, .{ .rel32 }, .{ 0x0f, 0x8e }, 0, .none, .none }, + .{ .jna, .d, .{ .rel32 }, .{ 0x0f, 0x86 }, 0, .none, .none }, + .{ .jnae, .d, .{ .rel32 }, .{ 0x0f, 0x82 }, 0, .none, .none }, + .{ .jnb, .d, .{ .rel32 }, .{ 0x0f, 0x83 }, 0, .none, .none }, + .{ .jnbe, .d, .{ .rel32 }, .{ 0x0f, 0x87 }, 0, .none, .none }, + .{ .jnc, .d, .{ .rel32 }, .{ 0x0f, 0x83 }, 0, .none, .none }, + .{ .jne, .d, .{ .rel32 }, .{ 0x0f, 0x85 }, 0, .none, .none }, + .{ .jng, .d, .{ .rel32 }, .{ 0x0f, 0x8e }, 0, .none, .none }, + .{ .jnge, .d, .{ .rel32 }, .{ 0x0f, 0x8c }, 0, .none, .none }, + .{ .jnl, .d, .{ .rel32 }, .{ 0x0f, 0x8d }, 0, .none, .none }, + .{ .jnle, .d, .{ .rel32 }, .{ 0x0f, 0x8f }, 0, .none, .none }, + .{ .jno, .d, .{ .rel32 }, .{ 0x0f, 0x81 }, 0, .none, .none }, + .{ .jnp, .d, .{ .rel32 }, .{ 0x0f, 0x8b }, 0, .none, .none }, + .{ .jns, .d, .{ .rel32 }, .{ 0x0f, 0x89 }, 0, .none, .none }, + .{ .jnz, .d, .{ .rel32 }, .{ 0x0f, 0x85 }, 0, .none, .none }, + .{ .jo, .d, .{ .rel32 }, .{ 0x0f, 0x80 }, 0, .none, .none }, + .{ .jp, .d, .{ .rel32 }, .{ 0x0f, 0x8a }, 0, .none, .none }, + .{ .jpe, .d, .{ .rel32 }, .{ 0x0f, 0x8a }, 0, .none, .none }, + .{ .jpo, .d, .{ .rel32 }, .{ 0x0f, 0x8b }, 0, .none, .none }, + .{ .js, .d, .{ .rel32 }, .{ 0x0f, 0x88 }, 0, .none, .none }, + .{ .jz, .d, .{ .rel32 }, .{ 0x0f, 0x84 }, 0, .none, .none }, + + .{ .jmp, .d, .{ .rel32 }, .{ 0xe9 }, 0, .none, .none }, + .{ .jmp, .m, .{ .rm64 }, .{ 0xff }, 4, .none, .none }, + + .{ .lahf, .z, .{}, .{ 0x9f }, 0, .none, .@"32bit" }, + .{ .lahf, .z, .{}, .{ 0x9f }, 0, .none, .sahf }, + + .{ .lar, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x02 }, 0, .none, .none }, + .{ .lar, .rm, .{ .r32, .r32_m16 }, .{ 0x0f, 0x02 }, 0, .none, .none }, + + .{ .lea, .rm, .{ .r16, .m }, .{ 0x8d }, 0, .short, .none }, + .{ .lea, .rm, .{ .r32, .m }, .{ 0x8d }, 0, .none, .none }, + .{ .lea, .rm, .{ .r64, .m }, .{ 0x8d }, 0, .long, .none }, + + .{ .leave, .z, .{}, .{ 0xc9 }, 0, .none, .none }, + + .{ .lfence, .z, .{}, .{ 0x0f, 0xae, 0xe8 }, 0, .none, .none }, + + .{ .lgdt, .m, .{ .m }, .{ 0x0f, 0x01 }, 2, .none, .none }, + .{ .lidt, .m, .{ .m }, .{ 0x0f, 0x01 }, 3, .none, .none }, + + .{ .lldt, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 2, .none, .none }, + + .{ .lmsw, .m, .{ .rm16 }, .{ 0x0f, 0x01 }, 6, .none, .none }, + + .{ .lods, .z, .{ .m8 }, .{ 0xac }, 0, .none, .none }, + .{ .lods, .z, .{ .m16 }, .{ 0xad }, 0, .short, .none }, + .{ .lods, .z, .{ .m32 }, .{ 0xad }, 0, .none, .none }, + .{ .lods, .z, .{ .m64 }, .{ 0xad }, 0, .long, .none }, + .{ .lodsb, .z, .{ }, .{ 0xac }, 0, .none, .none }, + .{ .lodsw, .z, .{ }, .{ 0xad }, 0, .short, .none }, + .{ .lodsd, .z, .{ }, .{ 0xad }, 0, .none, .none }, + .{ .lodsq, .z, .{ }, .{ 0xad }, 0, .long, .none }, + + .{ .loop, .d, .{ .rel8 }, .{ 0xe2 }, 0, .none, .none }, + .{ .loope, .d, .{ .rel8 }, .{ 0xe1 }, 0, .none, .none }, + .{ .loopne, .d, .{ .rel8 }, .{ 0xe0 }, 0, .none, .none }, + + .{ .lsl, .rm, .{ .r16, .rm16 }, .{ 0x0f, 0x03 }, 0, .none, .none }, + .{ .lsl, .rm, .{ .r32, .r32_m16 }, .{ 0x0f, 0x03 }, 0, .none, .none }, + .{ .lsl, .rm, .{ .r64, .r32_m16 }, .{ 0x0f, 0x03 }, 0, .none, .none }, + + .{ .ltr, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 3, .none, .none }, + + .{ .mfence, .z, .{}, .{ 0x0f, 0xae, 0xf0 }, 0, .none, .none }, + + .{ .mov, .mr, .{ .rm8, .r8 }, .{ 0x88 }, 0, .none, .none }, + .{ .mov, .mr, .{ .rm8, .r8 }, .{ 0x88 }, 0, .rex, .none }, + .{ .mov, .mr, .{ .rm16, .r16 }, .{ 0x89 }, 0, .short, .none }, + .{ .mov, .mr, .{ .rm32, .r32 }, .{ 0x89 }, 0, .none, .none }, + .{ .mov, .mr, .{ .rm64, .r64 }, .{ 0x89 }, 0, .long, .none }, + .{ .mov, .rm, .{ .r8, .rm8 }, .{ 0x8a }, 0, .none, .none }, + .{ .mov, .rm, .{ .r8, .rm8 }, .{ 0x8a }, 0, .rex, .none }, + .{ .mov, .rm, .{ .r16, .rm16 }, .{ 0x8b }, 0, .short, .none }, + .{ .mov, .rm, .{ .r32, .rm32 }, .{ 0x8b }, 0, .none, .none }, + .{ .mov, .rm, .{ .r64, .rm64 }, .{ 0x8b }, 0, .long, .none }, + .{ .mov, .mr, .{ .rm16, .sreg }, .{ 0x8c }, 0, .short, .none }, + .{ .mov, .mr, .{ .r32_m16, .sreg }, .{ 0x8c }, 0, .none, .none }, + .{ .mov, .mr, .{ .r64_m16, .sreg }, .{ 0x8c }, 0, .long, .none }, + .{ .mov, .rm, .{ .sreg, .rm16 }, .{ 0x8e }, 0, .short, .none }, + .{ .mov, .rm, .{ .sreg, .r32_m16 }, .{ 0x8e }, 0, .none, .none }, + .{ .mov, .rm, .{ .sreg, .r64_m16 }, .{ 0x8e }, 0, .long, .none }, + .{ .mov, .fd, .{ .al, .moffs }, .{ 0xa0 }, 0, .none, .none }, + .{ .mov, .fd, .{ .ax, .moffs }, .{ 0xa1 }, 0, .short, .none }, + .{ .mov, .fd, .{ .eax, .moffs }, .{ 0xa1 }, 0, .none, .none }, + .{ .mov, .fd, .{ .rax, .moffs }, .{ 0xa1 }, 0, .long, .none }, + .{ .mov, .td, .{ .moffs, .al }, .{ 0xa2 }, 0, .none, .none }, + .{ .mov, .td, .{ .moffs, .ax }, .{ 0xa3 }, 0, .short, .none }, + .{ .mov, .td, .{ .moffs, .eax }, .{ 0xa3 }, 0, .none, .none }, + .{ .mov, .td, .{ .moffs, .rax }, .{ 0xa3 }, 0, .long, .none }, + .{ .mov, .oi, .{ .r8, .imm8 }, .{ 0xb0 }, 0, .none, .none }, + .{ .mov, .oi, .{ .r8, .imm8 }, .{ 0xb0 }, 0, .rex, .none }, + .{ .mov, .oi, .{ .r16, .imm16 }, .{ 0xb8 }, 0, .short, .none }, + .{ .mov, .oi, .{ .r32, .imm32 }, .{ 0xb8 }, 0, .none, .none }, + .{ .mov, .oi, .{ .r64, .imm64 }, .{ 0xb8 }, 0, .long, .none }, + .{ .mov, .mi, .{ .rm8, .imm8 }, .{ 0xc6 }, 0, .none, .none }, + .{ .mov, .mi, .{ .rm8, .imm8 }, .{ 0xc6 }, 0, .rex, .none }, + .{ .mov, .mi, .{ .rm16, .imm16 }, .{ 0xc7 }, 0, .short, .none }, + .{ .mov, .mi, .{ .rm32, .imm32 }, .{ 0xc7 }, 0, .none, .none }, + .{ .mov, .mi, .{ .rm64, .imm32s }, .{ 0xc7 }, 0, .long, .none }, + + .{ .mov, .mr, .{ .r32, .cr }, .{ 0x0f, 0x20 }, 0, .none, .@"32bit" }, + .{ .mov, .mr, .{ .r64, .cr }, .{ 0x0f, 0x20 }, 0, .none, .@"64bit" }, + .{ .mov, .rm, .{ .cr, .r32 }, .{ 0x0f, 0x22 }, 0, .none, .@"32bit" }, + .{ .mov, .rm, .{ .cr, .r64 }, .{ 0x0f, 0x22 }, 0, .none, .@"64bit" }, + + .{ .mov, .mr, .{ .r32, .dr }, .{ 0x0f, 0x21 }, 0, .none, .@"32bit" }, + .{ .mov, .mr, .{ .r64, .dr }, .{ 0x0f, 0x21 }, 0, .none, .@"64bit" }, + .{ .mov, .rm, .{ .dr, .r32 }, .{ 0x0f, 0x23 }, 0, .none, .@"32bit" }, + .{ .mov, .rm, .{ .dr, .r64 }, .{ 0x0f, 0x23 }, 0, .none, .@"64bit" }, + + .{ .movbe, .rm, .{ .r16, .m16 }, .{ 0x0f, 0x38, 0xf0 }, 0, .short, .movbe }, + .{ .movbe, .rm, .{ .r32, .m32 }, .{ 0x0f, 0x38, 0xf0 }, 0, .none, .movbe }, + .{ .movbe, .rm, .{ .r64, .m64 }, .{ 0x0f, 0x38, 0xf0 }, 0, .long, .movbe }, + .{ .movbe, .mr, .{ .m16, .r16 }, .{ 0x0f, 0x38, 0xf1 }, 0, .short, .movbe }, + .{ .movbe, .mr, .{ .m32, .r32 }, .{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe }, + .{ .movbe, .mr, .{ .m64, .r64 }, .{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe }, + + .{ .movs, .z, .{ .m8, .m8 }, .{ 0xa4 }, 0, .none, .none }, + .{ .movs, .z, .{ .m16, .m16 }, .{ 0xa5 }, 0, .short, .none }, + .{ .movs, .z, .{ .m32, .m32 }, .{ 0xa5 }, 0, .none, .none }, + .{ .movs, .z, .{ .m64, .m64 }, .{ 0xa5 }, 0, .long, .none }, + .{ .movsb, .z, .{ }, .{ 0xa4 }, 0, .none, .none }, + .{ .movsw, .z, .{ }, .{ 0xa5 }, 0, .short, .none }, + .{ .movsd, .z, .{ }, .{ 0xa5 }, 0, .none, .none }, + .{ .movsq, .z, .{ }, .{ 0xa5 }, 0, .long, .none }, + + .{ .movsx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xbe }, 0, .short, .none }, + .{ .movsx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xbe }, 0, .rex_short, .none }, + .{ .movsx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xbe }, 0, .none, .none }, + .{ .movsx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xbe }, 0, .rex, .none }, + .{ .movsx, .rm, .{ .r64, .rm8 }, .{ 0x0f, 0xbe }, 0, .long, .none }, + .{ .movsx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xbf }, 0, .none, .none }, + .{ .movsx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xbf }, 0, .rex, .none }, + .{ .movsx, .rm, .{ .r64, .rm16 }, .{ 0x0f, 0xbf }, 0, .long, .none }, + + // This instruction is discouraged. + .{ .movsxd, .rm, .{ .r32, .rm32 }, .{ 0x63 }, 0, .none, .@"64bit" }, + .{ .movsxd, .rm, .{ .r64, .rm32 }, .{ 0x63 }, 0, .long, .@"64bit" }, + + .{ .movzx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xb6 }, 0, .short, .none }, + .{ .movzx, .rm, .{ .r16, .rm8 }, .{ 0x0f, 0xb6 }, 0, .rex_short, .none }, + .{ .movzx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xb6 }, 0, .none, .none }, + .{ .movzx, .rm, .{ .r32, .rm8 }, .{ 0x0f, 0xb6 }, 0, .rex, .none }, + .{ .movzx, .rm, .{ .r64, .rm8 }, .{ 0x0f, 0xb6 }, 0, .long, .none }, + .{ .movzx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xb7 }, 0, .none, .none }, + .{ .movzx, .rm, .{ .r32, .rm16 }, .{ 0x0f, 0xb7 }, 0, .rex, .none }, + .{ .movzx, .rm, .{ .r64, .rm16 }, .{ 0x0f, 0xb7 }, 0, .long, .none }, + + .{ .mul, .m, .{ .rm8 }, .{ 0xf6 }, 4, .none, .none }, + .{ .mul, .m, .{ .rm8 }, .{ 0xf6 }, 4, .rex, .none }, + .{ .mul, .m, .{ .rm16 }, .{ 0xf7 }, 4, .short, .none }, + .{ .mul, .m, .{ .rm32 }, .{ 0xf7 }, 4, .none, .none }, + .{ .mul, .m, .{ .rm64 }, .{ 0xf7 }, 4, .long, .none }, + + .{ .neg, .m, .{ .rm8 }, .{ 0xf6 }, 3, .none, .none }, + .{ .neg, .m, .{ .rm8 }, .{ 0xf6 }, 3, .rex, .none }, + .{ .neg, .m, .{ .rm16 }, .{ 0xf7 }, 3, .short, .none }, + .{ .neg, .m, .{ .rm32 }, .{ 0xf7 }, 3, .none, .none }, + .{ .neg, .m, .{ .rm64 }, .{ 0xf7 }, 3, .long, .none }, + + .{ .nop, .z, .{}, .{ 0x90 }, 0, .none, .none }, + + .{ .not, .m, .{ .rm8 }, .{ 0xf6 }, 2, .none, .none }, + .{ .not, .m, .{ .rm8 }, .{ 0xf6 }, 2, .rex, .none }, + .{ .not, .m, .{ .rm16 }, .{ 0xf7 }, 2, .short, .none }, + .{ .not, .m, .{ .rm32 }, .{ 0xf7 }, 2, .none, .none }, + .{ .not, .m, .{ .rm64 }, .{ 0xf7 }, 2, .long, .none }, + + .{ .@"or", .zi, .{ .al, .imm8 }, .{ 0x0c }, 0, .none, .none }, + .{ .@"or", .zi, .{ .ax, .imm16 }, .{ 0x0d }, 0, .short, .none }, + .{ .@"or", .zi, .{ .eax, .imm32 }, .{ 0x0d }, 0, .none, .none }, + .{ .@"or", .zi, .{ .rax, .imm32s }, .{ 0x0d }, 0, .long, .none }, + .{ .@"or", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 1, .none, .none }, + .{ .@"or", .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 1, .rex, .none }, + .{ .@"or", .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 1, .short, .none }, + .{ .@"or", .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 1, .none, .none }, + .{ .@"or", .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 1, .long, .none }, + .{ .@"or", .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 1, .short, .none }, + .{ .@"or", .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 1, .none, .none }, + .{ .@"or", .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 1, .long, .none }, + .{ .@"or", .mr, .{ .rm8, .r8 }, .{ 0x08 }, 0, .none, .none }, + .{ .@"or", .mr, .{ .rm8, .r8 }, .{ 0x08 }, 0, .rex, .none }, + .{ .@"or", .mr, .{ .rm16, .r16 }, .{ 0x09 }, 0, .short, .none }, + .{ .@"or", .mr, .{ .rm32, .r32 }, .{ 0x09 }, 0, .none, .none }, + .{ .@"or", .mr, .{ .rm64, .r64 }, .{ 0x09 }, 0, .long, .none }, + .{ .@"or", .rm, .{ .r8, .rm8 }, .{ 0x0a }, 0, .none, .none }, + .{ .@"or", .rm, .{ .r8, .rm8 }, .{ 0x0a }, 0, .rex, .none }, + .{ .@"or", .rm, .{ .r16, .rm16 }, .{ 0x0b }, 0, .short, .none }, + .{ .@"or", .rm, .{ .r32, .rm32 }, .{ 0x0b }, 0, .none, .none }, + .{ .@"or", .rm, .{ .r64, .rm64 }, .{ 0x0b }, 0, .long, .none }, + + .{ .out, .zi, .{ .imm8, .al }, .{ 0xe6 }, 0, .none, .none }, + .{ .out, .zi, .{ .imm8, .ax }, .{ 0xe7 }, 0, .short, .none }, + .{ .out, .zi, .{ .imm8, .eax }, .{ 0xe7 }, 0, .none, .none }, + .{ .out, .z, .{ .dx, .al }, .{ 0xee }, 0, .none, .none }, + .{ .out, .z, .{ .dx, .ax }, .{ 0xef }, 0, .short, .none }, + .{ .out, .z, .{ .dx, .eax }, .{ 0xef }, 0, .none, .none }, + + .{ .outs, .z, .{ .dx, .m8 }, .{ 0x6e }, 0, .none, .none }, + .{ .outs, .z, .{ .dx, .m16 }, .{ 0x6f }, 0, .short, .none }, + .{ .outs, .z, .{ .dx, .m32 }, .{ 0x6f }, 0, .none, .none }, + .{ .outsb, .z, .{ }, .{ 0x6e }, 0, .none, .none }, + .{ .outsw, .z, .{ }, .{ 0x6f }, 0, .short, .none }, + .{ .outsd, .z, .{ }, .{ 0x6f }, 0, .none, .none }, + + .{ .pause, .z, .{}, .{ 0xf3, 0x90 }, 0, .none, .none }, + + .{ .pop, .o, .{ .r16 }, .{ 0x58 }, 0, .short, .none }, + .{ .pop, .o, .{ .r64 }, .{ 0x58 }, 0, .none, .none }, + .{ .pop, .m, .{ .rm16 }, .{ 0x8f }, 0, .short, .none }, + .{ .pop, .m, .{ .rm64 }, .{ 0x8f }, 0, .none, .none }, + + .{ .popf, .z, .{}, .{ 0x9d }, 0, .short, .none }, + .{ .popfd, .z, .{}, .{ 0x9d }, 0, .none, .@"32bit" }, + .{ .popfq, .z, .{}, .{ 0x9d }, 0, .none, .@"64bit" }, + + .{ .push, .o, .{ .r16 }, .{ 0x50 }, 0, .short, .none }, + .{ .push, .o, .{ .r64 }, .{ 0x50 }, 0, .none, .none }, + .{ .push, .m, .{ .rm16 }, .{ 0xff }, 6, .short, .none }, + .{ .push, .m, .{ .rm64 }, .{ 0xff }, 6, .none, .none }, + .{ .push, .i, .{ .imm8 }, .{ 0x6a }, 0, .none, .none }, + .{ .push, .i, .{ .imm16 }, .{ 0x68 }, 0, .short, .none }, + .{ .push, .i, .{ .imm32 }, .{ 0x68 }, 0, .none, .none }, + + .{ .pushfq, .z, .{}, .{ 0x9c }, 0, .none, .none }, + + .{ .ret, .z, .{}, .{ 0xc3 }, 0, .none, .none }, + + .{ .rcl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 2, .none, .none }, + .{ .rcl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 2, .rex, .none }, + .{ .rcl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 2, .none, .none }, + .{ .rcl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 2, .rex, .none }, + .{ .rcl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 2, .none, .none }, + .{ .rcl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 2, .rex, .none }, + .{ .rcl, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 2, .short, .none }, + .{ .rcl, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 2, .short, .none }, + .{ .rcl, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 2, .short, .none }, + .{ .rcl, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 2, .none, .none }, + .{ .rcl, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 2, .long, .none }, + .{ .rcl, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 2, .none, .none }, + .{ .rcl, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 2, .long, .none }, + .{ .rcl, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 2, .none, .none }, + .{ .rcl, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 2, .long, .none }, + + .{ .rcr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 3, .none, .none }, + .{ .rcr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 3, .rex, .none }, + .{ .rcr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 3, .none, .none }, + .{ .rcr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 3, .rex, .none }, + .{ .rcr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 3, .none, .none }, + .{ .rcr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 3, .rex, .none }, + .{ .rcr, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 3, .short, .none }, + .{ .rcr, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 3, .short, .none }, + .{ .rcr, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 3, .short, .none }, + .{ .rcr, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 3, .none, .none }, + .{ .rcr, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 3, .long, .none }, + .{ .rcr, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 3, .none, .none }, + .{ .rcr, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 3, .long, .none }, + .{ .rcr, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 3, .none, .none }, + .{ .rcr, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 3, .long, .none }, + + .{ .rdfsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 0, .none, .fsgsbase }, + .{ .rdfsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 0, .long, .fsgsbase }, + .{ .rdgsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 1, .none, .fsgsbase }, + .{ .rdgsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 1, .long, .fsgsbase }, + + .{ .rdmsr, .z, .{}, .{ 0x0f, 0x32 }, 0, .none, .none }, + + .{ .rdpid, .m, .{ .r32 }, .{ 0xf3, 0x0f, 0xc7 }, 7, .none, .@"rdpid 32bit" }, + .{ .rdpid, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xc7 }, 7, .none, .@"rdpid 64bit" }, + + .{ .rdpkru, .z, .{}, .{ 0x0f, 0x01, 0xee }, 0, .none, .pku }, + + .{ .rdpmc, .z, .{}, .{ 0x0f, 0x33 }, 0, .none, .none }, + + .{ .rdrand, .m, .{ .r16 }, .{ 0x0f, 0xc7 }, 6, .short, .rdrnd }, + .{ .rdrand, .m, .{ .r32 }, .{ 0x0f, 0xc7 }, 6, .none, .rdrnd }, + .{ .rdrand, .m, .{ .r64 }, .{ 0x0f, 0xc7 }, 6, .long, .rdrnd }, + + .{ .rdseed, .m, .{ .r16 }, .{ 0x0f, 0xc7 }, 7, .short, .rdseed }, + .{ .rdseed, .m, .{ .r32 }, .{ 0x0f, 0xc7 }, 7, .none, .rdseed }, + .{ .rdseed, .m, .{ .r64 }, .{ 0x0f, 0xc7 }, 7, .long, .rdseed }, + + .{ .rdsspd, .m, .{ .r32 }, .{ 0xf3, 0x0f, 0x1e }, 1, .none, .shstk }, + .{ .rdsspq, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0x1e }, 1, .long, .shstk }, + + .{ .rdtsc, .z, .{}, .{ 0x0f, 0x31 }, 0, .none, .none }, + + .{ .rdtscp, .z, .{}, .{ 0x0f, 0x01, 0xf9 }, 0, .none, .none }, + + .{ .rol, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 0, .none, .none }, + .{ .rol, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 0, .rex, .none }, + .{ .rol, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 0, .none, .none }, + .{ .rol, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 0, .rex, .none }, + .{ .rol, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 0, .none, .none }, + .{ .rol, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 0, .rex, .none }, + .{ .rol, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 0, .short, .none }, + .{ .rol, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 0, .short, .none }, + .{ .rol, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 0, .short, .none }, + .{ .rol, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 0, .none, .none }, + .{ .rol, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 0, .long, .none }, + .{ .rol, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 0, .none, .none }, + .{ .rol, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 0, .long, .none }, + .{ .rol, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 0, .none, .none }, + .{ .rol, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 0, .long, .none }, + + .{ .ror, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 1, .none, .none }, + .{ .ror, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 1, .rex, .none }, + .{ .ror, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 1, .none, .none }, + .{ .ror, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 1, .rex, .none }, + .{ .ror, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 1, .none, .none }, + .{ .ror, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 1, .rex, .none }, + .{ .ror, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 1, .short, .none }, + .{ .ror, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 1, .short, .none }, + .{ .ror, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 1, .short, .none }, + .{ .ror, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 1, .none, .none }, + .{ .ror, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 1, .long, .none }, + .{ .ror, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 1, .none, .none }, + .{ .ror, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 1, .long, .none }, + .{ .ror, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 1, .none, .none }, + .{ .ror, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 1, .long, .none }, + + .{ .rsm, .z, .{}, .{ 0x0f, 0xaa }, 0, .none, .none }, + + .{ .sahf, .z, .{}, .{ 0x9e }, 0, .none, .@"32bit" }, + .{ .sahf, .z, .{}, .{ 0x9e }, 0, .none, .sahf }, + + .{ .sal, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .none, .none }, + .{ .sal, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .rex, .none }, + .{ .sal, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 4, .short, .none }, + .{ .sal, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 4, .none, .none }, + .{ .sal, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 4, .long, .none }, + .{ .sal, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .none, .none }, + .{ .sal, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .rex, .none }, + .{ .sal, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 4, .short, .none }, + .{ .sal, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 4, .none, .none }, + .{ .sal, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 4, .long, .none }, + .{ .sal, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .none, .none }, + .{ .sal, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .rex, .none }, + .{ .sal, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 4, .short, .none }, + .{ .sal, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 4, .none, .none }, + .{ .sal, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 4, .long, .none }, + + .{ .sar, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 7, .none, .none }, + .{ .sar, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 7, .rex, .none }, + .{ .sar, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 7, .short, .none }, + .{ .sar, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 7, .none, .none }, + .{ .sar, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 7, .long, .none }, + .{ .sar, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 7, .none, .none }, + .{ .sar, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 7, .rex, .none }, + .{ .sar, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 7, .short, .none }, + .{ .sar, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 7, .none, .none }, + .{ .sar, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 7, .long, .none }, + .{ .sar, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 7, .none, .none }, + .{ .sar, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 7, .rex, .none }, + .{ .sar, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 7, .short, .none }, + .{ .sar, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 7, .none, .none }, + .{ .sar, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 7, .long, .none }, + + .{ .sbb, .zi, .{ .al, .imm8 }, .{ 0x1c }, 0, .none, .none }, + .{ .sbb, .zi, .{ .ax, .imm16 }, .{ 0x1d }, 0, .short, .none }, + .{ .sbb, .zi, .{ .eax, .imm32 }, .{ 0x1d }, 0, .none, .none }, + .{ .sbb, .zi, .{ .rax, .imm32s }, .{ 0x1d }, 0, .long, .none }, + .{ .sbb, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 3, .none, .none }, + .{ .sbb, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 3, .rex, .none }, + .{ .sbb, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 3, .short, .none }, + .{ .sbb, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 3, .none, .none }, + .{ .sbb, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 3, .long, .none }, + .{ .sbb, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 3, .short, .none }, + .{ .sbb, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 3, .none, .none }, + .{ .sbb, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 3, .long, .none }, + .{ .sbb, .mr, .{ .rm8, .r8 }, .{ 0x18 }, 0, .none, .none }, + .{ .sbb, .mr, .{ .rm8, .r8 }, .{ 0x18 }, 0, .rex, .none }, + .{ .sbb, .mr, .{ .rm16, .r16 }, .{ 0x19 }, 0, .short, .none }, + .{ .sbb, .mr, .{ .rm32, .r32 }, .{ 0x19 }, 0, .none, .none }, + .{ .sbb, .mr, .{ .rm64, .r64 }, .{ 0x19 }, 0, .long, .none }, + .{ .sbb, .rm, .{ .r8, .rm8 }, .{ 0x1a }, 0, .none, .none }, + .{ .sbb, .rm, .{ .r8, .rm8 }, .{ 0x1a }, 0, .rex, .none }, + .{ .sbb, .rm, .{ .r16, .rm16 }, .{ 0x1b }, 0, .short, .none }, + .{ .sbb, .rm, .{ .r32, .rm32 }, .{ 0x1b }, 0, .none, .none }, + .{ .sbb, .rm, .{ .r64, .rm64 }, .{ 0x1b }, 0, .long, .none }, + + .{ .scas, .z, .{ .m8 }, .{ 0xae }, 0, .none, .none }, + .{ .scas, .z, .{ .m16 }, .{ 0xaf }, 0, .short, .none }, + .{ .scas, .z, .{ .m32 }, .{ 0xaf }, 0, .none, .none }, + .{ .scas, .z, .{ .m64 }, .{ 0xaf }, 0, .long, .none }, + .{ .scasb, .z, .{ }, .{ 0xae }, 0, .none, .none }, + .{ .scasw, .z, .{ }, .{ 0xaf }, 0, .short, .none }, + .{ .scasd, .z, .{ }, .{ 0xaf }, 0, .none, .none }, + .{ .scasq, .z, .{ }, .{ 0xaf }, 0, .long, .none }, + + .{ .senduipi, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xc7 }, 6, .none, .uintr }, + + .{ .serialize, .z, .{}, .{ 0x0f, 0x01, 0xe8 }, 0, .none, .serialize }, + + .{ .seta, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .none, .none }, + .{ .seta, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .rex, .none }, + .{ .setae, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .none, .none }, + .{ .setae, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .rex, .none }, + .{ .setb, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .none, .none }, + .{ .setb, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .rex, .none }, + .{ .setbe, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .none, .none }, + .{ .setbe, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .rex, .none }, + .{ .setc, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .none, .none }, + .{ .setc, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .rex, .none }, + .{ .sete, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .none, .none }, + .{ .sete, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .rex, .none }, + .{ .setg, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .none, .none }, + .{ .setg, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .rex, .none }, + .{ .setge, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .none, .none }, + .{ .setge, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .rex, .none }, + .{ .setl, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .none, .none }, + .{ .setl, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .rex, .none }, + .{ .setle, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .none, .none }, + .{ .setle, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .rex, .none }, + .{ .setna, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .none, .none }, + .{ .setna, .m, .{ .rm8 }, .{ 0x0f, 0x96 }, 0, .rex, .none }, + .{ .setnae, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .none, .none }, + .{ .setnae, .m, .{ .rm8 }, .{ 0x0f, 0x92 }, 0, .rex, .none }, + .{ .setnb, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .none, .none }, + .{ .setnb, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .rex, .none }, + .{ .setnbe, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .none, .none }, + .{ .setnbe, .m, .{ .rm8 }, .{ 0x0f, 0x97 }, 0, .rex, .none }, + .{ .setnc, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .none, .none }, + .{ .setnc, .m, .{ .rm8 }, .{ 0x0f, 0x93 }, 0, .rex, .none }, + .{ .setne, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .none, .none }, + .{ .setne, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .rex, .none }, + .{ .setng, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .none, .none }, + .{ .setng, .m, .{ .rm8 }, .{ 0x0f, 0x9e }, 0, .rex, .none }, + .{ .setnge, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .none, .none }, + .{ .setnge, .m, .{ .rm8 }, .{ 0x0f, 0x9c }, 0, .rex, .none }, + .{ .setnl, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .none, .none }, + .{ .setnl, .m, .{ .rm8 }, .{ 0x0f, 0x9d }, 0, .rex, .none }, + .{ .setnle, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .none, .none }, + .{ .setnle, .m, .{ .rm8 }, .{ 0x0f, 0x9f }, 0, .rex, .none }, + .{ .setno, .m, .{ .rm8 }, .{ 0x0f, 0x91 }, 0, .none, .none }, + .{ .setno, .m, .{ .rm8 }, .{ 0x0f, 0x91 }, 0, .rex, .none }, + .{ .setnp, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .none, .none }, + .{ .setnp, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .rex, .none }, + .{ .setns, .m, .{ .rm8 }, .{ 0x0f, 0x99 }, 0, .none, .none }, + .{ .setns, .m, .{ .rm8 }, .{ 0x0f, 0x99 }, 0, .rex, .none }, + .{ .setnz, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .none, .none }, + .{ .setnz, .m, .{ .rm8 }, .{ 0x0f, 0x95 }, 0, .rex, .none }, + .{ .seto, .m, .{ .rm8 }, .{ 0x0f, 0x90 }, 0, .none, .none }, + .{ .seto, .m, .{ .rm8 }, .{ 0x0f, 0x90 }, 0, .rex, .none }, + .{ .setp, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .none, .none }, + .{ .setp, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .rex, .none }, + .{ .setpe, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .none, .none }, + .{ .setpe, .m, .{ .rm8 }, .{ 0x0f, 0x9a }, 0, .rex, .none }, + .{ .setpo, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .none, .none }, + .{ .setpo, .m, .{ .rm8 }, .{ 0x0f, 0x9b }, 0, .rex, .none }, + .{ .sets, .m, .{ .rm8 }, .{ 0x0f, 0x98 }, 0, .none, .none }, + .{ .sets, .m, .{ .rm8 }, .{ 0x0f, 0x98 }, 0, .rex, .none }, + .{ .setz, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .none, .none }, + .{ .setz, .m, .{ .rm8 }, .{ 0x0f, 0x94 }, 0, .rex, .none }, + + .{ .sfence, .z, .{}, .{ 0x0f, 0xae, 0xf8 }, 0, .none, .none }, + + .{ .sidt, .m, .{ .m }, .{ 0x0f, 0x01 }, 1, .none, .none }, + + .{ .sldt, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 0, .none, .none }, + + .{ .smsw, .m, .{ .rm16 }, .{ 0x0f, 0x01 }, 4, .short, .none }, + .{ .smsw, .m, .{ .r32_m16 }, .{ 0x0f, 0x01 }, 4, .none, .none }, + .{ .smsw, .m, .{ .r64_m16 }, .{ 0x0f, 0x01 }, 4, .long, .none }, + + .{ .shl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .none, .none }, + .{ .shl, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 4, .rex, .none }, + .{ .shl, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 4, .short, .none }, + .{ .shl, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 4, .none, .none }, + .{ .shl, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 4, .long, .none }, + .{ .shl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .none, .none }, + .{ .shl, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 4, .rex, .none }, + .{ .shl, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 4, .short, .none }, + .{ .shl, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 4, .none, .none }, + .{ .shl, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 4, .long, .none }, + .{ .shl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .none, .none }, + .{ .shl, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 4, .rex, .none }, + .{ .shl, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 4, .short, .none }, + .{ .shl, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 4, .none, .none }, + .{ .shl, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 4, .long, .none }, + + .{ .shld, .mri, .{ .rm16, .r16, .imm8 }, .{ 0x0f, 0xa4 }, 0, .short, .none }, + .{ .shld, .mrc, .{ .rm16, .r16, .cl }, .{ 0x0f, 0xa5 }, 0, .short, .none }, + .{ .shld, .mri, .{ .rm32, .r32, .imm8 }, .{ 0x0f, 0xa4 }, 0, .none, .none }, + .{ .shld, .mri, .{ .rm64, .r64, .imm8 }, .{ 0x0f, 0xa4 }, 0, .long, .none }, + .{ .shld, .mrc, .{ .rm32, .r32, .cl }, .{ 0x0f, 0xa5 }, 0, .none, .none }, + .{ .shld, .mrc, .{ .rm64, .r64, .cl }, .{ 0x0f, 0xa5 }, 0, .long, .none }, + + .{ .shr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 5, .none, .none }, + .{ .shr, .m1, .{ .rm8, .unity }, .{ 0xd0 }, 5, .rex, .none }, + .{ .shr, .m1, .{ .rm16, .unity }, .{ 0xd1 }, 5, .short, .none }, + .{ .shr, .m1, .{ .rm32, .unity }, .{ 0xd1 }, 5, .none, .none }, + .{ .shr, .m1, .{ .rm64, .unity }, .{ 0xd1 }, 5, .long, .none }, + .{ .shr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 5, .none, .none }, + .{ .shr, .mc, .{ .rm8, .cl }, .{ 0xd2 }, 5, .rex, .none }, + .{ .shr, .mc, .{ .rm16, .cl }, .{ 0xd3 }, 5, .short, .none }, + .{ .shr, .mc, .{ .rm32, .cl }, .{ 0xd3 }, 5, .none, .none }, + .{ .shr, .mc, .{ .rm64, .cl }, .{ 0xd3 }, 5, .long, .none }, + .{ .shr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 5, .none, .none }, + .{ .shr, .mi, .{ .rm8, .imm8 }, .{ 0xc0 }, 5, .rex, .none }, + .{ .shr, .mi, .{ .rm16, .imm8 }, .{ 0xc1 }, 5, .short, .none }, + .{ .shr, .mi, .{ .rm32, .imm8 }, .{ 0xc1 }, 5, .none, .none }, + .{ .shr, .mi, .{ .rm64, .imm8 }, .{ 0xc1 }, 5, .long, .none }, + + .{ .shrd, .mri, .{ .rm16, .r16, .imm8 }, .{ 0x0f, 0xac }, 0, .short, .none }, + .{ .shrd, .mrc, .{ .rm16, .r16, .cl }, .{ 0x0f, 0xad }, 0, .short, .none }, + .{ .shrd, .mri, .{ .rm32, .r32, .imm8 }, .{ 0x0f, 0xac }, 0, .none, .none }, + .{ .shrd, .mri, .{ .rm64, .r64, .imm8 }, .{ 0x0f, 0xac }, 0, .long, .none }, + .{ .shrd, .mrc, .{ .rm32, .r32, .cl }, .{ 0x0f, 0xad }, 0, .none, .none }, + .{ .shrd, .mrc, .{ .rm64, .r64, .cl }, .{ 0x0f, 0xad }, 0, .long, .none }, + + .{ .stac, .z, .{}, .{ 0x0f, 0x01, 0xcb }, 0, .none, .smap }, + + .{ .stc, .z, .{}, .{ 0xf9 }, 0, .none, .none }, + + .{ .std, .z, .{}, .{ 0xfd }, 0, .none, .none }, + + .{ .sti, .z, .{}, .{ 0xfb }, 0, .none, .none }, + + .{ .str, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 1, .none, .none }, + + .{ .stui, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr }, + + .{ .stos, .z, .{ .m8 }, .{ 0xaa }, 0, .none, .none }, + .{ .stos, .z, .{ .m16 }, .{ 0xab }, 0, .short, .none }, + .{ .stos, .z, .{ .m32 }, .{ 0xab }, 0, .none, .none }, + .{ .stos, .z, .{ .m64 }, .{ 0xab }, 0, .long, .none }, + .{ .stosb, .z, .{ }, .{ 0xaa }, 0, .none, .none }, + .{ .stosw, .z, .{ }, .{ 0xab }, 0, .short, .none }, + .{ .stosd, .z, .{ }, .{ 0xab }, 0, .none, .none }, + .{ .stosq, .z, .{ }, .{ 0xab }, 0, .long, .none }, + + .{ .sub, .zi, .{ .al, .imm8 }, .{ 0x2c }, 0, .none, .none }, + .{ .sub, .zi, .{ .ax, .imm16 }, .{ 0x2d }, 0, .short, .none }, + .{ .sub, .zi, .{ .eax, .imm32 }, .{ 0x2d }, 0, .none, .none }, + .{ .sub, .zi, .{ .rax, .imm32s }, .{ 0x2d }, 0, .long, .none }, + .{ .sub, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 5, .none, .none }, + .{ .sub, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 5, .rex, .none }, + .{ .sub, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 5, .short, .none }, + .{ .sub, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 5, .none, .none }, + .{ .sub, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 5, .long, .none }, + .{ .sub, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 5, .short, .none }, + .{ .sub, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 5, .none, .none }, + .{ .sub, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 5, .long, .none }, + .{ .sub, .mr, .{ .rm8, .r8 }, .{ 0x28 }, 0, .none, .none }, + .{ .sub, .mr, .{ .rm8, .r8 }, .{ 0x28 }, 0, .rex, .none }, + .{ .sub, .mr, .{ .rm16, .r16 }, .{ 0x29 }, 0, .short, .none }, + .{ .sub, .mr, .{ .rm32, .r32 }, .{ 0x29 }, 0, .none, .none }, + .{ .sub, .mr, .{ .rm64, .r64 }, .{ 0x29 }, 0, .long, .none }, + .{ .sub, .rm, .{ .r8, .rm8 }, .{ 0x2a }, 0, .none, .none }, + .{ .sub, .rm, .{ .r8, .rm8 }, .{ 0x2a }, 0, .rex, .none }, + .{ .sub, .rm, .{ .r16, .rm16 }, .{ 0x2b }, 0, .short, .none }, + .{ .sub, .rm, .{ .r32, .rm32 }, .{ 0x2b }, 0, .none, .none }, + .{ .sub, .rm, .{ .r64, .rm64 }, .{ 0x2b }, 0, .long, .none }, + + .{ .swapgs, .z, .{}, .{ 0x0f, 0x01, 0xf8 }, 0, .none, .@"64bit" }, + + .{ .syscall, .z, .{}, .{ 0x0f, 0x05 }, 0, .none, .@"64bit" }, + + .{ .sysenter, .z, .{}, .{ 0x0f, 0x34 }, 0, .none, .none }, + + .{ .sysexit, .z, .{}, .{ 0x0f, 0x35 }, 0, .none, .none }, + .{ .sysexit, .z, .{}, .{ 0x0f, 0x35 }, 0, .long, .none }, + + .{ .sysret, .z, .{}, .{ 0x0f, 0x37 }, 0, .none, .none }, + .{ .sysret, .z, .{}, .{ 0x0f, 0x37 }, 0, .long, .none }, + + .{ .@"test", .zi, .{ .al, .imm8 }, .{ 0xa8 }, 0, .none, .none }, + .{ .@"test", .zi, .{ .ax, .imm16 }, .{ 0xa9 }, 0, .short, .none }, + .{ .@"test", .zi, .{ .eax, .imm32 }, .{ 0xa9 }, 0, .none, .none }, + .{ .@"test", .zi, .{ .rax, .imm32s }, .{ 0xa9 }, 0, .long, .none }, + .{ .@"test", .mi, .{ .rm8, .imm8 }, .{ 0xf6 }, 0, .none, .none }, + .{ .@"test", .mi, .{ .rm8, .imm8 }, .{ 0xf6 }, 0, .rex, .none }, + .{ .@"test", .mi, .{ .rm16, .imm16 }, .{ 0xf7 }, 0, .short, .none }, + .{ .@"test", .mi, .{ .rm32, .imm32 }, .{ 0xf7 }, 0, .none, .none }, + .{ .@"test", .mi, .{ .rm64, .imm32s }, .{ 0xf7 }, 0, .long, .none }, + .{ .@"test", .mr, .{ .rm8, .r8 }, .{ 0x84 }, 0, .none, .none }, + .{ .@"test", .mr, .{ .rm8, .r8 }, .{ 0x84 }, 0, .rex, .none }, + .{ .@"test", .mr, .{ .rm16, .r16 }, .{ 0x85 }, 0, .short, .none }, + .{ .@"test", .mr, .{ .rm32, .r32 }, .{ 0x85 }, 0, .none, .none }, + .{ .@"test", .mr, .{ .rm64, .r64 }, .{ 0x85 }, 0, .long, .none }, + + .{ .testui, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xed }, 0, .none, .uintr }, + + .{ .tpause, .m, .{ .r32 }, .{ 0x66, 0x0f, 0xae }, 6, .none, .waitpkg }, + + .{ .ud0, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xff }, 0, .none, .none }, + .{ .ud1, .rm, .{ .r32, .rm32 }, .{ 0x0f, 0xb9 }, 0, .none, .none }, + .{ .ud2, .z, .{}, .{ 0x0f, 0x0b }, 0, .none, .none }, + + .{ .uiret, .z, .{}, .{ 0xf3, 0x0f, 0x01, 0xec }, 0, .none, .uintr }, + + .{ .umonitor, .m, .{ .r64 }, .{ 0xf3, 0x0f, 0xae }, 6, .none, .waitpkg }, + + .{ .umwait, .m, .{ .r32 }, .{ 0xf2, 0x0f, 0xae }, 6, .none, .waitpkg }, + + .{ .verr, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 4, .none, .none }, + .{ .verw, .m, .{ .rm16 }, .{ 0x0f, 0x00 }, 5, .none, .none }, + + .{ .wrfsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 2, .none, .fsgsbase }, + .{ .wrfsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 2, .long, .fsgsbase }, + .{ .wrgsbase, .m, .{ .r32 }, .{ 0xf3 ,0x0f, 0xae }, 3, .none, .fsgsbase }, + .{ .wrgsbase, .m, .{ .r64 }, .{ 0xf3 ,0x0f, 0xae }, 3, .long, .fsgsbase }, + + .{ .wrmsr, .z, .{}, .{ 0x0f, 0x30 }, 0, .none, .none }, + + .{ .wrpkru, .z, .{}, .{ 0x0f, 0x01, 0xef }, 0, .none, .pku }, + + .{ .wrssd, .mr, .{ .m32, .r32 }, .{ 0x0f, 0x38, 0xf6 }, 0, .none, .shstk }, + .{ .wrssq, .mr, .{ .m64, .r64 }, .{ 0x0f, 0x38, 0xf6 }, 0, .long, .shstk }, + + .{ .wrussd, .mr, .{ .m32, .r32 }, .{ 0x66, 0x0f, 0x38, 0xf5 }, 0, .none, .shstk }, + .{ .wrussq, .mr, .{ .m64, .r64 }, .{ 0x66, 0x0f, 0x38, 0xf5 }, 0, .long, .shstk }, + + .{ .xadd, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xc0 }, 0, .none, .none }, + .{ .xadd, .mr, .{ .rm8, .r8 }, .{ 0x0f, 0xc0 }, 0, .rex, .none }, + .{ .xadd, .mr, .{ .rm16, .r16 }, .{ 0x0f, 0xc1 }, 0, .short, .none }, + .{ .xadd, .mr, .{ .rm32, .r32 }, .{ 0x0f, 0xc1 }, 0, .none, .none }, + .{ .xadd, .mr, .{ .rm64, .r64 }, .{ 0x0f, 0xc1 }, 0, .long, .none }, + + .{ .xchg, .zo, .{ .ax, .r16 }, .{ 0x90 }, 0, .short, .none }, + .{ .xchg, .oz, .{ .r16, .ax }, .{ 0x90 }, 0, .short, .none }, + .{ .xchg, .zo, .{ .eax, .r32 }, .{ 0x90 }, 0, .none, .none }, + .{ .xchg, .zo, .{ .rax, .r64 }, .{ 0x90 }, 0, .long, .none }, + .{ .xchg, .oz, .{ .r32, .eax }, .{ 0x90 }, 0, .none, .none }, + .{ .xchg, .oz, .{ .r64, .rax }, .{ 0x90 }, 0, .long, .none }, + .{ .xchg, .mr, .{ .rm8, .r8 }, .{ 0x86 }, 0, .none, .none }, + .{ .xchg, .mr, .{ .rm8, .r8 }, .{ 0x86 }, 0, .rex, .none }, + .{ .xchg, .rm, .{ .r8, .rm8 }, .{ 0x86 }, 0, .none, .none }, + .{ .xchg, .rm, .{ .r8, .rm8 }, .{ 0x86 }, 0, .rex, .none }, + .{ .xchg, .mr, .{ .rm16, .r16 }, .{ 0x87 }, 0, .short, .none }, + .{ .xchg, .rm, .{ .r16, .rm16 }, .{ 0x87 }, 0, .short, .none }, + .{ .xchg, .mr, .{ .rm32, .r32 }, .{ 0x87 }, 0, .none, .none }, + .{ .xchg, .mr, .{ .rm64, .r64 }, .{ 0x87 }, 0, .long, .none }, + .{ .xchg, .rm, .{ .r32, .rm32 }, .{ 0x87 }, 0, .none, .none }, + .{ .xchg, .rm, .{ .r64, .rm64 }, .{ 0x87 }, 0, .long, .none }, + + .{ .xgetbv, .z, .{}, .{ 0x0f, 0x01, 0xd0 }, 0, .none, .none }, + + .{ .xlat, .z, .{ .m8 }, .{ 0xd7 }, 0, .none, .@"32bit" }, + .{ .xlat, .z, .{ .m8 }, .{ 0xd7 }, 0, .long, .@"64bit" }, + .{ .xlatb, .z, .{ }, .{ 0xd7 }, 0, .none, .@"32bit" }, + .{ .xlatb, .z, .{ }, .{ 0xd7 }, 0, .long, .@"64bit" }, + + .{ .xor, .zi, .{ .al, .imm8 }, .{ 0x34 }, 0, .none, .none }, + .{ .xor, .zi, .{ .ax, .imm16 }, .{ 0x35 }, 0, .short, .none }, + .{ .xor, .zi, .{ .eax, .imm32 }, .{ 0x35 }, 0, .none, .none }, + .{ .xor, .zi, .{ .rax, .imm32s }, .{ 0x35 }, 0, .long, .none }, + .{ .xor, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 6, .none, .none }, + .{ .xor, .mi, .{ .rm8, .imm8 }, .{ 0x80 }, 6, .rex, .none }, + .{ .xor, .mi, .{ .rm16, .imm16 }, .{ 0x81 }, 6, .short, .none }, + .{ .xor, .mi, .{ .rm32, .imm32 }, .{ 0x81 }, 6, .none, .none }, + .{ .xor, .mi, .{ .rm64, .imm32s }, .{ 0x81 }, 6, .long, .none }, + .{ .xor, .mi, .{ .rm16, .imm8s }, .{ 0x83 }, 6, .short, .none }, + .{ .xor, .mi, .{ .rm32, .imm8s }, .{ 0x83 }, 6, .none, .none }, + .{ .xor, .mi, .{ .rm64, .imm8s }, .{ 0x83 }, 6, .long, .none }, + .{ .xor, .mr, .{ .rm8, .r8 }, .{ 0x30 }, 0, .none, .none }, + .{ .xor, .mr, .{ .rm8, .r8 }, .{ 0x30 }, 0, .rex, .none }, + .{ .xor, .mr, .{ .rm16, .r16 }, .{ 0x31 }, 0, .short, .none }, + .{ .xor, .mr, .{ .rm32, .r32 }, .{ 0x31 }, 0, .none, .none }, + .{ .xor, .mr, .{ .rm64, .r64 }, .{ 0x31 }, 0, .long, .none }, + .{ .xor, .rm, .{ .r8, .rm8 }, .{ 0x32 }, 0, .none, .none }, + .{ .xor, .rm, .{ .r8, .rm8 }, .{ 0x32 }, 0, .rex, .none }, + .{ .xor, .rm, .{ .r16, .rm16 }, .{ 0x33 }, 0, .short, .none }, + .{ .xor, .rm, .{ .r32, .rm32 }, .{ 0x33 }, 0, .none, .none }, + .{ .xor, .rm, .{ .r64, .rm64 }, .{ 0x33 }, 0, .long, .none }, + + // X87 + .{ .f2xm1, .z, .{}, .{ 0xd9, 0xf0 }, 0, .none, .x87 }, + + .{ .fabs, .z, .{}, .{ 0xd9, 0xe1 }, 0, .none, .x87 }, + + .{ .fadd, .m, .{ .m32 }, .{ 0xd8 }, 0, .none, .x87 }, + .{ .fadd, .m, .{ .m64 }, .{ 0xdc }, 0, .none, .x87 }, + .{ .fadd, .zo, .{ .st0, .st }, .{ 0xd8, 0xc0 }, 0, .none, .x87 }, + .{ .fadd, .oz, .{ .st, .st0 }, .{ 0xdc, 0xc0 }, 0, .none, .x87 }, + .{ .faddp, .oz, .{ .st, .st0 }, .{ 0xde, 0xc0 }, 0, .none, .x87 }, + .{ .faddp, .z, .{ }, .{ 0xde, 0xc1 }, 0, .none, .x87 }, + .{ .fiadd, .m, .{ .m32 }, .{ 0xda }, 0, .none, .x87 }, + .{ .fiadd, .m, .{ .m16 }, .{ 0xde }, 0, .none, .x87 }, + + .{ .fbld, .m, .{ .m80 }, .{ 0xdf }, 4, .none, .x87 }, + + .{ .fbstp, .m, .{ .m80 }, .{ 0xdf }, 6, .none, .x87 }, + + .{ .fchs, .z, .{}, .{ 0xd9, 0xe0 }, 0, .none, .x87 }, + + .{ .fclex, .z, .{}, .{ 0xdb, 0xe2 }, 0, .wait, .x87 }, + .{ .fnclex, .z, .{}, .{ 0xdb, 0xe2 }, 0, .none, .x87 }, + + .{ .fcmovb, .zo, .{ .st0, .st }, .{ 0xda, 0xc0 }, 0, .none, .@"cmov x87" }, + .{ .fcmove, .zo, .{ .st0, .st }, .{ 0xda, 0xc8 }, 0, .none, .@"cmov x87" }, + .{ .fcmovbe, .zo, .{ .st0, .st }, .{ 0xda, 0xd0 }, 0, .none, .@"cmov x87" }, + .{ .fcmovu, .zo, .{ .st0, .st }, .{ 0xda, 0xd8 }, 0, .none, .@"cmov x87" }, + .{ .fcmovnb, .zo, .{ .st0, .st }, .{ 0xdb, 0xc0 }, 0, .none, .@"cmov x87" }, + .{ .fcmovne, .zo, .{ .st0, .st }, .{ 0xdb, 0xc8 }, 0, .none, .@"cmov x87" }, + .{ .fcmovnbe, .zo, .{ .st0, .st }, .{ 0xdb, 0xd0 }, 0, .none, .@"cmov x87" }, + .{ .fcmovnu, .zo, .{ .st0, .st }, .{ 0xdb, 0xd8 }, 0, .none, .@"cmov x87" }, + + .{ .fcom, .m, .{ .m32 }, .{ 0xd8 }, 2, .none, .x87 }, + .{ .fcom, .m, .{ .m64 }, .{ 0xdc }, 2, .none, .x87 }, + .{ .fcom, .o, .{ .st }, .{ 0xd8, 0xd0 }, 0, .none, .x87 }, + .{ .fcom, .z, .{ }, .{ 0xd8, 0xd1 }, 0, .none, .x87 }, + .{ .fcomp, .m, .{ .m32 }, .{ 0xd8 }, 3, .none, .x87 }, + .{ .fcomp, .m, .{ .m64 }, .{ 0xdc }, 3, .none, .x87 }, + .{ .fcomp, .o, .{ .st }, .{ 0xd8, 0xd8 }, 0, .none, .x87 }, + .{ .fcomp, .z, .{ }, .{ 0xd8, 0xd9 }, 0, .none, .x87 }, + .{ .fcompp, .z, .{ }, .{ 0xde, 0xd9 }, 0, .none, .x87 }, + + .{ .fcomi, .zo, .{ .st0, .st }, .{ 0xdb, 0xf0 }, 0, .none, .@"cmov x87" }, + .{ .fcomip, .zo, .{ .st0, .st }, .{ 0xdf, 0xf0 }, 0, .none, .@"cmov x87" }, + .{ .fucomi, .zo, .{ .st0, .st }, .{ 0xdb, 0xe8 }, 0, .none, .@"cmov x87" }, + .{ .fucomip, .zo, .{ .st0, .st }, .{ 0xdf, 0xe8 }, 0, .none, .@"cmov x87" }, + + .{ .fcos, .z, .{}, .{ 0xd9, 0xff }, 0, .none, .x87 }, + + .{ .fdecstp, .z, .{}, .{ 0xd9, 0xf6 }, 0, .none, .x87 }, + + .{ .fdiv, .m, .{ .m32 }, .{ 0xd8 }, 6, .none, .x87 }, + .{ .fdiv, .m, .{ .m64 }, .{ 0xdc }, 6, .none, .x87 }, + .{ .fdiv, .zo, .{ .st0, .st }, .{ 0xd8, 0xf0 }, 0, .none, .x87 }, + .{ .fdiv, .oz, .{ .st, .st0 }, .{ 0xdc, 0xf8 }, 0, .none, .x87 }, + .{ .fdivp, .oz, .{ .st, .st0 }, .{ 0xde, 0xf8 }, 0, .none, .x87 }, + .{ .fdivp, .z, .{ }, .{ 0xde, 0xf9 }, 0, .none, .x87 }, + .{ .fidiv, .m, .{ .m32 }, .{ 0xda }, 6, .none, .x87 }, + .{ .fidiv, .m, .{ .m16 }, .{ 0xde }, 6, .none, .x87 }, + + .{ .fdivr, .m, .{ .m32 }, .{ 0xd8 }, 7, .none, .x87 }, + .{ .fdivr, .m, .{ .m64 }, .{ 0xdc }, 7, .none, .x87 }, + .{ .fdivr, .zo, .{ .st0, .st }, .{ 0xd8, 0xf8 }, 0, .none, .x87 }, + .{ .fdivr, .oz, .{ .st, .st0 }, .{ 0xdc, 0xf0 }, 0, .none, .x87 }, + .{ .fdivrp, .oz, .{ .st, .st0 }, .{ 0xde, 0xf0 }, 0, .none, .x87 }, + .{ .fdivrp, .z, .{ }, .{ 0xde, 0xf1 }, 0, .none, .x87 }, + .{ .fidivr, .m, .{ .m32 }, .{ 0xda }, 7, .none, .x87 }, + .{ .fidivr, .m, .{ .m16 }, .{ 0xde }, 7, .none, .x87 }, + + .{ .ffree, .o, .{ .st }, .{ 0xdd, 0xc0 }, 0, .none, .x87 }, + + .{ .ficom, .m, .{ .m16 }, .{ 0xde }, 2, .none, .x87 }, + .{ .ficom, .m, .{ .m32 }, .{ 0xda }, 2, .none, .x87 }, + .{ .ficomp, .m, .{ .m16 }, .{ 0xde }, 3, .none, .x87 }, + .{ .ficomp, .m, .{ .m32 }, .{ 0xda }, 3, .none, .x87 }, + + .{ .fild, .m, .{ .m16 }, .{ 0xdf }, 0, .none, .x87 }, + .{ .fild, .m, .{ .m32 }, .{ 0xdb }, 0, .none, .x87 }, + .{ .fild, .m, .{ .m64 }, .{ 0xdf }, 5, .none, .x87 }, + + .{ .fincstp, .z, .{}, .{ 0xd9, 0xf7 }, 0, .none, .x87 }, + + .{ .finit, .z, .{}, .{ 0xdb, 0xe3 }, 0, .wait, .x87 }, + .{ .fninit, .z, .{}, .{ 0xdb, 0xe3 }, 0, .none, .x87 }, + + .{ .fist, .m, .{ .m16 }, .{ 0xdf }, 2, .none, .x87 }, + .{ .fist, .m, .{ .m32 }, .{ 0xdb }, 2, .none, .x87 }, + .{ .fistp, .m, .{ .m16 }, .{ 0xdf }, 3, .none, .x87 }, + .{ .fistp, .m, .{ .m32 }, .{ 0xdb }, 3, .none, .x87 }, + .{ .fistp, .m, .{ .m64 }, .{ 0xdf }, 7, .none, .x87 }, + + .{ .fld, .m, .{ .m32 }, .{ 0xd9 }, 0, .none, .x87 }, + .{ .fld, .m, .{ .m64 }, .{ 0xdd }, 0, .none, .x87 }, + .{ .fld, .m, .{ .m80 }, .{ 0xdb }, 5, .none, .x87 }, + .{ .fld, .o, .{ .st }, .{ 0xd9, 0xc0 }, 0, .none, .x87 }, + + .{ .fld1, .z, .{}, .{ 0xd9, 0xe8 }, 0, .none, .x87 }, + .{ .fldl2t, .z, .{}, .{ 0xd9, 0xe9 }, 0, .none, .x87 }, + .{ .fldl2e, .z, .{}, .{ 0xd9, 0xea }, 0, .none, .x87 }, + .{ .fldpi, .z, .{}, .{ 0xd9, 0xeb }, 0, .none, .x87 }, + .{ .fldlg2, .z, .{}, .{ 0xd9, 0xec }, 0, .none, .x87 }, + .{ .fldln2, .z, .{}, .{ 0xd9, 0xed }, 0, .none, .x87 }, + .{ .fldz, .z, .{}, .{ 0xd9, 0xee }, 0, .none, .x87 }, + + .{ .fldcw, .m, .{ .m16 }, .{ 0xd9 }, 5, .none, .x87 }, + + .{ .fldenv, .m, .{ .m }, .{ 0xd9 }, 4, .none, .x87 }, + + .{ .fmul, .m, .{ .m32 }, .{ 0xd8 }, 1, .none, .x87 }, + .{ .fmul, .m, .{ .m64 }, .{ 0xdc }, 1, .none, .x87 }, + .{ .fmul, .zo, .{ .st0, .st }, .{ 0xd8, 0xc8 }, 0, .none, .x87 }, + .{ .fmul, .oz, .{ .st, .st0 }, .{ 0xdc, 0xc8 }, 0, .none, .x87 }, + .{ .fmulp, .oz, .{ .st, .st0 }, .{ 0xde, 0xc8 }, 0, .none, .x87 }, + .{ .fmulp, .z, .{ }, .{ 0xde, 0xc9 }, 0, .none, .x87 }, + .{ .fimul, .m, .{ .m32 }, .{ 0xda }, 1, .none, .x87 }, + .{ .fimul, .m, .{ .m16 }, .{ 0xde }, 1, .none, .x87 }, + + .{ .fnop, .z, .{}, .{ 0xd9, 0xd0 }, 0, .none, .x87 }, + + .{ .fpatan, .z, .{}, .{ 0xd9, 0xf3 }, 0, .none, .x87 }, + + .{ .fprem, .z, .{}, .{ 0xd9, 0xf8 }, 0, .none, .x87 }, + + .{ .fprem1, .z, .{}, .{ 0xd9, 0xf5 }, 0, .none, .x87 }, + + .{ .fptan, .z, .{}, .{ 0xd9, 0xf2 }, 0, .none, .x87 }, + + .{ .frndint, .z, .{}, .{ 0xd9, 0xfc }, 0, .none, .x87 }, + + .{ .frstor, .m, .{ .m }, .{ 0xdd }, 4, .none, .x87 }, + + .{ .fsave, .m, .{ .m }, .{ 0xdd }, 6, .wait, .x87 }, + .{ .fnsave, .m, .{ .m }, .{ 0xdd }, 6, .none, .x87 }, + + .{ .fscale, .z, .{}, .{ 0xd9, 0xfd }, 0, .none, .x87 }, + + .{ .fsin, .z, .{}, .{ 0xd9, 0xfe }, 0, .none, .x87 }, + + .{ .fsincos, .z, .{}, .{ 0xd9, 0xfb }, 0, .none, .x87 }, + + .{ .fsqrt, .z, .{}, .{ 0xd9, 0xfa }, 0, .none, .x87 }, + + .{ .fst, .m, .{ .m32 }, .{ 0xd9 }, 2, .none, .x87 }, + .{ .fst, .m, .{ .m64 }, .{ 0xdd }, 2, .none, .x87 }, + .{ .fst, .o, .{ .st }, .{ 0xdd, 0xd0 }, 0, .none, .x87 }, + .{ .fstp, .m, .{ .m32 }, .{ 0xd9 }, 3, .none, .x87 }, + .{ .fstp, .m, .{ .m64 }, .{ 0xdd }, 3, .none, .x87 }, + .{ .fstp, .m, .{ .m80 }, .{ 0xdb }, 7, .none, .x87 }, + .{ .fstp, .o, .{ .st }, .{ 0xdd, 0xd8 }, 0, .none, .x87 }, + + .{ .fstcw, .m, .{ .m16 }, .{ 0xd9 }, 7, .wait, .x87 }, + .{ .fnstcw, .m, .{ .m16 }, .{ 0xd9 }, 7, .none, .x87 }, + + .{ .fstenv, .m, .{ .m }, .{ 0xd9 }, 6, .wait, .x87 }, + .{ .fnstenv, .m, .{ .m }, .{ 0xd9 }, 6, .none, .x87 }, + + .{ .fstsw, .m, .{ .m16 }, .{ 0xdd }, 7, .wait, .x87 }, + .{ .fstsw, .m, .{ .ax }, .{ 0xdf }, 4, .wait, .x87 }, + .{ .fnstsw, .m, .{ .m16 }, .{ 0xdd }, 7, .none, .x87 }, + .{ .fnstsw, .m, .{ .ax }, .{ 0xdf }, 4, .none, .x87 }, + + .{ .fsub, .m, .{ .m32 }, .{ 0xd8 }, 4, .none, .x87 }, + .{ .fsub, .m, .{ .m64 }, .{ 0xdc }, 4, .none, .x87 }, + .{ .fsub, .zo, .{ .st0, .st }, .{ 0xd8, 0xe0 }, 0, .none, .x87 }, + .{ .fsub, .oz, .{ .st, .st0 }, .{ 0xdc, 0xe8 }, 0, .none, .x87 }, + .{ .fsubp, .oz, .{ .st, .st0 }, .{ 0xde, 0xe8 }, 0, .none, .x87 }, + .{ .fsubp, .z, .{ }, .{ 0xde, 0xe9 }, 0, .none, .x87 }, + .{ .fisub, .m, .{ .m32 }, .{ 0xda }, 4, .none, .x87 }, + .{ .fisub, .m, .{ .m16 }, .{ 0xde }, 4, .none, .x87 }, + + .{ .fsubr, .m, .{ .m32 }, .{ 0xd8 }, 5, .none, .x87 }, + .{ .fsubr, .m, .{ .m64 }, .{ 0xdc }, 5, .none, .x87 }, + .{ .fsubr, .zo, .{ .st0, .st }, .{ 0xd8, 0xe8 }, 0, .none, .x87 }, + .{ .fsubr, .oz, .{ .st, .st0 }, .{ 0xdc, 0xe0 }, 0, .none, .x87 }, + .{ .fsubrp, .oz, .{ .st, .st0 }, .{ 0xde, 0xe0 }, 0, .none, .x87 }, + .{ .fsubrp, .z, .{ }, .{ 0xde, 0xe1 }, 0, .none, .x87 }, + .{ .fisubr, .m, .{ .m32 }, .{ 0xda }, 5, .none, .x87 }, + .{ .fisubr, .m, .{ .m16 }, .{ 0xde }, 5, .none, .x87 }, + + .{ .ftst, .z, .{}, .{ 0xd9, 0xe4 }, 0, .none, .x87 }, + + .{ .fucom, .o, .{ .st }, .{ 0xdd, 0xe0 }, 0, .none, .x87 }, + .{ .fucom, .z, .{ }, .{ 0xdd, 0xe1 }, 0, .none, .x87 }, + .{ .fucomp, .o, .{ .st }, .{ 0xdd, 0xe8 }, 0, .none, .x87 }, + .{ .fucomp, .z, .{ }, .{ 0xdd, 0xe9 }, 0, .none, .x87 }, + .{ .fucompp, .z, .{ }, .{ 0xda, 0xe9 }, 0, .none, .x87 }, + + .{ .fxam, .z, .{}, .{ 0xd9, 0xe5 }, 0, .none, .x87 }, + + .{ .fxch, .o, .{ .st }, .{ 0xd9, 0xc8 }, 0, .none, .x87 }, + .{ .fxch, .z, .{ }, .{ 0xd9, 0xc9 }, 0, .none, .x87 }, + + .{ .fxtract, .z, .{}, .{ 0xd9, 0xf4 }, 0, .none, .x87 }, + + .{ .fyl2x, .z, .{}, .{ 0xd9, 0xf1 }, 0, .none, .x87 }, + + .{ .fyl2xp1, .z, .{}, .{ 0xd9, 0xf9 }, 0, .none, .x87 }, + + .{ .wait, .z, .{}, .{ 0x9b }, 0, .none, .x87 }, + .{ .fwait, .z, .{}, .{ 0x9b }, 0, .none, .x87 }, + + // MMX + .{ .emms, .z, .{}, .{ 0x0f, 0x77 }, 0, .none, .mmx }, + + // SSE + .{ .addps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x58 }, 0, .none, .sse }, + + .{ .addss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse }, + + .{ .andnps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x55 }, 0, .none, .sse }, + + .{ .andps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x54 }, 0, .none, .sse }, + + .{ .cmpps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc2 }, 0, .none, .sse }, + + .{ .cmpss, .rmi, .{ .xmm, .xmm_m32, .imm8 }, .{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse }, + + .{ .comiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2f }, 0, .none, .sse }, + + .{ .cvtpi2ps, .rm, .{ .xmm, .mm_m64 }, .{ 0x0f, 0x2a }, 0, .none, .sse }, + + .{ .cvtps2pi, .rm, .{ .mm, .xmm_m64 }, .{ 0x0f, 0x2d }, 0, .none, .sse }, + + .{ .cvtsi2ss, .rm, .{ .xmm, .rm32 }, .{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse }, + .{ .cvtsi2ss, .rm, .{ .xmm, .rm64 }, .{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse }, + + .{ .cvtss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .none, .sse }, + .{ .cvtss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .long, .sse }, + + .{ .cvttps2pi, .rm, .{ .mm, .xmm_m64 }, .{ 0x0f, 0x2c }, 0, .none, .sse }, + + .{ .cvttss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .none, .sse }, + .{ .cvttss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .long, .sse }, + + .{ .divps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5e }, 0, .none, .sse }, + + .{ .divss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse }, + + .{ .fxrstor, .m, .{ .m }, .{ 0x0f, 0xae }, 1, .none, .fxsr }, + .{ .fxrstor64, .m, .{ .m }, .{ 0x0f, 0xae }, 1, .long, .fxsr }, + + .{ .fxsave, .m, .{ .m }, .{ 0x0f, 0xae }, 0, .none, .fxsr }, + .{ .fxsave64, .m, .{ .m }, .{ 0x0f, 0xae }, 0, .long, .fxsr }, + + .{ .ldmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 2, .none, .sse }, + + .{ .maxps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5f }, 0, .none, .sse }, + + .{ .maxss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5f }, 0, .none, .sse }, + + .{ .minps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5d }, 0, .none, .sse }, + + .{ .minss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5d }, 0, .none, .sse }, + + .{ .movaps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x28 }, 0, .none, .sse }, + .{ .movaps, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x29 }, 0, .none, .sse }, + + .{ .movhlps, .rm, .{ .xmm, .xmm }, .{ 0x0f, 0x12 }, 0, .none, .sse }, + + .{ .movhps, .rm, .{ .xmm, .m64 }, .{ 0x0f, 0x16 }, 0, .none, .sse }, + .{ .movhps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x17 }, 0, .none, .sse }, + + .{ .movlhps, .rm, .{ .xmm, .xmm }, .{ 0x0f, 0x16 }, 0, .none, .sse }, + + .{ .movlps, .rm, .{ .xmm, .m64 }, .{ 0x0f, 0x12 }, 0, .none, .sse }, + .{ .movlps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x13 }, 0, .none, .sse }, + + .{ .movmskps, .rm, .{ .r32, .xmm }, .{ 0x0f, 0x50 }, 0, .none, .sse }, + .{ .movmskps, .rm, .{ .r64, .xmm }, .{ 0x0f, 0x50 }, 0, .none, .sse }, + + .{ .movss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse }, + .{ .movss, .mr, .{ .xmm_m32, .xmm }, .{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse }, + + .{ .movups, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x10 }, 0, .none, .sse }, + .{ .movups, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x11 }, 0, .none, .sse }, + + .{ .mulps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x59 }, 0, .none, .sse }, + + .{ .mulss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x59 }, 0, .none, .sse }, + + .{ .orps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x56 }, 0, .none, .sse }, + + .{ .pmovmskb, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse }, + .{ .pmovmskb, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse }, + + .{ .prefetchit0, .m, .{ .mrip8 }, .{ 0x0f, 0x18 }, 7, .none, .@"prefetchi 64bit" }, + .{ .prefetchit1, .m, .{ .mrip8 }, .{ 0x0f, 0x18 }, 6, .none, .@"prefetchi 64bit" }, + + .{ .prefetchnta, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 0, .none, .prefetch }, + .{ .prefetcht0, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 1, .none, .prefetch }, + .{ .prefetcht1, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 2, .none, .prefetch }, + .{ .prefetcht2, .m, .{ .m8 }, .{ 0x0f, 0x18 }, 3, .none, .prefetch }, + + .{ .prefetchw, .m, .{ .m8 }, .{ 0x0f, 0x0d }, 1, .none, .prfchw }, + + .{ .prefetchwt1, .m, .{ .m8 }, .{ 0x0f, 0x0d }, 2, .none, .prefetchwt1 }, + + .{ .shufps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc6 }, 0, .none, .sse }, + + .{ .sqrtps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x51 }, 0, .none, .sse }, + + .{ .sqrtss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse }, + + .{ .stmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 3, .none, .sse }, + + .{ .subps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5c }, 0, .none, .sse }, + + .{ .subss, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse }, + + .{ .ucomiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2e }, 0, .none, .sse }, + + .{ .unpckhps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x15 }, 0, .none, .sse }, + + .{ .unpcklps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x14 }, 0, .none, .sse }, + + .{ .xorps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x57 }, 0, .none, .sse }, + + // SSE2 + .{ .addpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x58 }, 0, .none, .sse2 }, + + .{ .addsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x58 }, 0, .none, .sse2 }, + + .{ .andnpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x55 }, 0, .none, .sse2 }, + + .{ .andpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 }, + + .{ .cmppd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc2 }, 0, .none, .sse2 }, + + .{ .cmpsd, .rmi, .{ .xmm, .xmm_m64, .imm8 }, .{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 }, + + .{ .comisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2f }, 0, .none, .sse2 }, + + .{ .cvtdq2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0xe6 }, 0, .none, .sse2 }, + + .{ .cvtdq2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5b }, 0, .none, .sse2 }, + + .{ .cvtpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xe6 }, 0, .none, .sse2 }, + + .{ .cvtpd2pi, .rm, .{ .mm, .xmm_m128 }, .{ 0x66, 0x0f, 0x2d }, 0, .none, .sse2 }, + + .{ .cvtpd2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5a }, 0, .none, .sse2 }, + + .{ .cvtpi2pd, .rm, .{ .xmm, .mm_m64 }, .{ 0x66, 0x0f, 0x2a }, 0, .none, .sse2 }, + + .{ .cvtps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5b }, 0, .none, .sse2 }, + + .{ .cvtps2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x0f, 0x5a }, 0, .none, .sse2 }, + + .{ .cvtsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .none, .sse2 }, + .{ .cvtsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .long, .sse2 }, + + .{ .cvtsd2ss, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5a }, 0, .none, .sse2 }, + + .{ .cvtsi2sd, .rm, .{ .xmm, .rm32 }, .{ 0xf2, 0x0f, 0x2a }, 0, .none, .sse2 }, + .{ .cvtsi2sd, .rm, .{ .xmm, .rm64 }, .{ 0xf2, 0x0f, 0x2a }, 0, .long, .sse2 }, + + .{ .cvtss2sd, .rm, .{ .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 }, + + .{ .cvttpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe6 }, 0, .none, .sse2 }, + + .{ .cvttpd2pi, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x2c }, 0, .none, .sse2 }, + + .{ .cvttps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x5b }, 0, .none, .sse2 }, + + .{ .cvttsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .none, .sse2 }, + .{ .cvttsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .long, .sse2 }, + + .{ .divpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5e }, 0, .none, .sse2 }, + + .{ .divsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 }, + + .{ .gf2p8affineinvqb, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xcf }, 0, .none, .gfni }, + + .{ .gf2p8affineqb, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xce }, 0, .none, .gfni }, + + .{ .gf2p8mulb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xcf }, 0, .none, .gfni }, + + .{ .maxpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5f }, 0, .none, .sse2 }, + + .{ .maxsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5f }, 0, .none, .sse2 }, + + .{ .minpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5d }, 0, .none, .sse2 }, + + .{ .minsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5d }, 0, .none, .sse2 }, + + .{ .movapd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x28 }, 0, .none, .sse2 }, + .{ .movapd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x29 }, 0, .none, .sse2 }, + + .{ .movd, .rm, .{ .xmm, .rm32 }, .{ 0x66, 0x0f, 0x6e }, 0, .none, .sse2 }, + .{ .movq, .rm, .{ .xmm, .rm64 }, .{ 0x66, 0x0f, 0x6e }, 0, .long, .sse2 }, + .{ .movd, .mr, .{ .rm32, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 }, + .{ .movq, .mr, .{ .rm64, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .long, .sse2 }, + + .{ .movdqa, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6f }, 0, .none, .sse2 }, + .{ .movdqa, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x7f }, 0, .none, .sse2 }, + + .{ .movdqu, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x6f }, 0, .none, .sse2 }, + .{ .movdqu, .mr, .{ .xmm_m128, .xmm }, .{ 0xf3, 0x0f, 0x7f }, 0, .none, .sse2 }, + + .{ .movhpd, .rm, .{ .xmm, .m64 }, .{ 0x66, 0x0f, 0x16 }, 0, .none, .sse2 }, + .{ .movhpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x17 }, 0, .none, .sse2 }, + + .{ .movlpd, .rm, .{ .xmm, .m64 }, .{ 0x66, 0x0f, 0x12 }, 0, .none, .sse2 }, + .{ .movlpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x13 }, 0, .none, .sse2 }, + + .{ .movmskpd, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 }, + .{ .movmskpd, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .none, .sse2 }, + + .{ .movsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x10 }, 0, .none, .sse2 }, + .{ .movsd, .mr, .{ .xmm_m64, .xmm }, .{ 0xf2, 0x0f, 0x11 }, 0, .none, .sse2 }, + + .{ .movq, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0x7e }, 0, .none, .sse2 }, + .{ .movq, .mr, .{ .xmm_m64, .xmm }, .{ 0x66, 0x0f, 0xd6 }, 0, .none, .sse2 }, + + .{ .movupd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x10 }, 0, .none, .sse2 }, + .{ .movupd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x11 }, 0, .none, .sse2 }, + + .{ .mulpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x59 }, 0, .none, .sse2 }, + + .{ .mulsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x59 }, 0, .none, .sse2 }, + + .{ .orpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 }, + + .{ .packsswb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x63 }, 0, .none, .sse2 }, + .{ .packssdw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6b }, 0, .none, .sse2 }, + + .{ .packuswb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x67 }, 0, .none, .sse2 }, + + .{ .paddb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfc }, 0, .none, .sse2 }, + .{ .paddw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfd }, 0, .none, .sse2 }, + .{ .paddd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfe }, 0, .none, .sse2 }, + .{ .paddq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd4 }, 0, .none, .sse2 }, + + .{ .paddsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xec }, 0, .none, .sse2 }, + .{ .paddsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xed }, 0, .none, .sse2 }, + + .{ .paddusb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdc }, 0, .none, .sse2 }, + .{ .paddusw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdd }, 0, .none, .sse2 }, + + .{ .pand, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdb }, 0, .none, .sse2 }, + + .{ .pandn, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdf }, 0, .none, .sse2 }, + + .{ .pcmpeqb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x74 }, 0, .none, .sse2 }, + .{ .pcmpeqw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x75 }, 0, .none, .sse2 }, + .{ .pcmpeqd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x76 }, 0, .none, .sse2 }, + + .{ .pcmpgtb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x64 }, 0, .none, .sse2 }, + .{ .pcmpgtw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x65 }, 0, .none, .sse2 }, + .{ .pcmpgtd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x66 }, 0, .none, .sse2 }, + + .{ .pextrw, .rmi, .{ .r32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 }, + + .{ .pinsrw, .rmi, .{ .xmm, .r32_m16, .imm8 }, .{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 }, + + .{ .pmaddwd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf5 }, 0, .none, .sse2 }, + + .{ .pmaxsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xee }, 0, .none, .sse2 }, + + .{ .pmaxub, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xde }, 0, .none, .sse2 }, + + .{ .pminsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xea }, 0, .none, .sse2 }, + + .{ .pminub, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xda }, 0, .none, .sse2 }, + + .{ .pmulhuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe4 }, 0, .none, .sse2 }, + + .{ .pmulhw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe5 }, 0, .none, .sse2 }, + + .{ .pmullw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd5 }, 0, .none, .sse2 }, + + .{ .pmuludq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf4 }, 0, .none, .sse2 }, + + .{ .por, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xeb }, 0, .none, .sse2 }, + + .{ .psadbw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf6 }, 0, .none, .sse2 }, + + .{ .pshufd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .none, .sse2 }, + + .{ .pshufhw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf3, 0x0f, 0x70 }, 0, .none, .sse2 }, + + .{ .pshuflw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 }, + + .{ .psllw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf1 }, 0, .none, .sse2 }, + .{ .psllw, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 6, .none, .sse2 }, + .{ .pslld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf2 }, 0, .none, .sse2 }, + .{ .pslld, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 6, .none, .sse2 }, + .{ .psllq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf3 }, 0, .none, .sse2 }, + .{ .psllq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 6, .none, .sse2 }, + + .{ .pslldq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 7, .none, .sse2 }, + + .{ .psraw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe1 }, 0, .none, .sse2 }, + .{ .psraw, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 4, .none, .sse2 }, + .{ .psrad, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe2 }, 0, .none, .sse2 }, + .{ .psrad, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 4, .none, .sse2 }, + + .{ .psrlw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 }, + .{ .psrlw, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 }, + .{ .psrld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd2 }, 0, .none, .sse2 }, + .{ .psrld, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 2, .none, .sse2 }, + .{ .psrlq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 }, + .{ .psrlq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 }, + + .{ .psrldq, .mi, .{ .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 3, .none, .sse2 }, + + .{ .psubb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf8 }, 0, .none, .sse2 }, + .{ .psubw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf9 }, 0, .none, .sse2 }, + .{ .psubd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfa }, 0, .none, .sse2 }, + + .{ .psubsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe8 }, 0, .none, .sse2 }, + .{ .psubsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe9 }, 0, .none, .sse2 }, + + .{ .psubq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfb }, 0, .none, .sse2 }, + + .{ .psubusb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd8 }, 0, .none, .sse2 }, + .{ .psubusw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd9 }, 0, .none, .sse2 }, + + .{ .punpckhbw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 }, + .{ .punpckhwd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 }, + .{ .punpckhdq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 }, + .{ .punpckhqdq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6d }, 0, .none, .sse2 }, + + .{ .punpcklbw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 }, + .{ .punpcklwd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 }, + .{ .punpckldq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 }, + .{ .punpcklqdq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 }, + + .{ .pxor, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xef }, 0, .none, .sse2 }, + + .{ .shufpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc6 }, 0, .none, .sse2 }, + + .{ .sqrtpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 }, + + .{ .sqrtsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 }, + + .{ .subpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5c }, 0, .none, .sse2 }, + + .{ .subsd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 }, + + .{ .ucomisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2e }, 0, .none, .sse2 }, + + .{ .unpckhpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x15 }, 0, .none, .sse2 }, + + .{ .unpcklpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x14 }, 0, .none, .sse2 }, + + .{ .xorpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x57 }, 0, .none, .sse2 }, + + // SSE3 + .{ .addsubpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd0 }, 0, .none, .sse3 }, + + .{ .addsubps, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .none, .sse3 }, + + .{ .fisttp, .m, .{ .m16 }, .{ 0xdf }, 1, .none, .@"sse3 x87" }, + .{ .fisttp, .m, .{ .m32 }, .{ 0xdb }, 1, .none, .@"sse3 x87" }, + .{ .fisttp, .m, .{ .m64 }, .{ 0xdd }, 1, .none, .@"sse3 x87" }, + + .{ .haddpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x7c }, 0, .none, .sse3 }, + + .{ .haddps, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0x7c }, 0, .none, .sse3 }, + + .{ .lddqu, .rm, .{ .xmm, .m128 }, .{ 0xf2, 0x0f, 0xf0 }, 0, .none, .sse3 }, + + .{ .movddup, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x12 }, 0, .none, .sse3 }, + + .{ .movshdup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x16 }, 0, .none, .sse3 }, + + .{ .movsldup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 }, + + // SSSE3 + .{ .pabsb, .rm, .{ .mm, .mm_m64 }, .{ 0x0f, 0x38, 0x1c }, 0, .none, .ssse3 }, + .{ .pabsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1c }, 0, .none, .ssse3 }, + .{ .pabsd, .rm, .{ .mm, .mm_m64 }, .{ 0x0f, 0x38, 0x1e }, 0, .none, .ssse3 }, + .{ .pabsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1e }, 0, .none, .ssse3 }, + .{ .pabsw, .rm, .{ .mm, .mm_m64 }, .{ 0x0f, 0x38, 0x1d }, 0, .none, .ssse3 }, + .{ .pabsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1d }, 0, .none, .ssse3 }, + + .{ .palignr, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 }, + + .{ .phaddw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x01 }, 0, .none, .ssse3 }, + .{ .phaddd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x02 }, 0, .none, .ssse3 }, + + .{ .phaddsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x03 }, 0, .none, .ssse3 }, + + .{ .phsubw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x05 }, 0, .none, .ssse3 }, + .{ .phsubd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x06 }, 0, .none, .ssse3 }, + + .{ .phsubsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x07 }, 0, .none, .ssse3 }, + + .{ .pmaddubsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x04 }, 0, .none, .ssse3 }, + + .{ .pmulhrsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0b }, 0, .none, .ssse3 }, + + .{ .pshufb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 }, + + .{ .psignb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x08 }, 0, .none, .ssse3 }, + .{ .psignw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x09 }, 0, .none, .ssse3 }, + .{ .psignd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0a }, 0, .none, .ssse3 }, + + // SSE4.1 + .{ .blendpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 }, + + .{ .blendps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .none, .sse4_1 }, + + .{ .blendvpd, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x15 }, 0, .none, .sse4_1 }, + .{ .blendvpd, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x66, 0x0f, 0x38, 0x15 }, 0, .none, .sse4_1 }, + + .{ .blendvps, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x14 }, 0, .none, .sse4_1 }, + .{ .blendvps, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x66, 0x0f, 0x38, 0x14 }, 0, .none, .sse4_1 }, + + .{ .dppd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x41 }, 0, .none, .sse4_1 }, + + .{ .dpps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x40 }, 0, .none, .sse4_1 }, + + .{ .extractps, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .none, .sse4_1 }, + + .{ .insertps, .rmi, .{ .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .none, .sse4_1 }, + + .{ .packusdw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 }, + + .{ .pblendvb, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 }, + .{ .pblendvb, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 }, + + .{ .pblendw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .none, .sse4_1 }, + + .{ .pcmpeqq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 }, + + .{ .pextrb, .mri, .{ .r32_m8, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 }, + .{ .pextrd, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 }, + .{ .pextrq, .mri, .{ .rm64, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 }, + + .{ .pextrw, .mri, .{ .r32_m16, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 }, + + .{ .phminposuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x41 }, 0, .none, .sse4_1 }, + + .{ .pinsrb, .rmi, .{ .xmm, .r32_m8, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .none, .sse4_1 }, + .{ .pinsrd, .rmi, .{ .xmm, .rm32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 }, + .{ .pinsrq, .rmi, .{ .xmm, .rm64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 }, + + .{ .pmaxsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3c }, 0, .none, .sse4_1 }, + .{ .pmaxsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3d }, 0, .none, .sse4_1 }, + + .{ .pmaxuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3e }, 0, .none, .sse4_1 }, + + .{ .pmaxud, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3f }, 0, .none, .sse4_1 }, + + .{ .pminsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x38 }, 0, .none, .sse4_1 }, + .{ .pminsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x39 }, 0, .none, .sse4_1 }, + + .{ .pminuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3a }, 0, .none, .sse4_1 }, + + .{ .pminud, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 }, + + .{ .pmovsxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x20 }, 0, .none, .sse4_1 }, + .{ .pmovsxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x21 }, 0, .none, .sse4_1 }, + .{ .pmovsxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x22 }, 0, .none, .sse4_1 }, + .{ .pmovsxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x23 }, 0, .none, .sse4_1 }, + .{ .pmovsxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x24 }, 0, .none, .sse4_1 }, + .{ .pmovsxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x25 }, 0, .none, .sse4_1 }, + + .{ .pmovzxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x30 }, 0, .none, .sse4_1 }, + .{ .pmovzxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x31 }, 0, .none, .sse4_1 }, + .{ .pmovzxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x32 }, 0, .none, .sse4_1 }, + .{ .pmovzxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x33 }, 0, .none, .sse4_1 }, + .{ .pmovzxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 }, + .{ .pmovzxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 }, + + .{ .pmuldq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .none, .sse4_1 }, + + .{ .pmulld, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 }, + + .{ .ptest, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .none, .sse4_1 }, + + .{ .roundpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 }, + + .{ .roundps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 }, + + .{ .roundsd, .rmi, .{ .xmm, .xmm_m64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 }, + + .{ .roundss, .rmi, .{ .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 }, + + // SSE4.2 + .{ .crc32, .rm, .{ .r32, .rm8 }, .{ 0xf2, 0x0f, 0x38, 0xf0 }, 0, .none, .crc32 }, + .{ .crc32, .rm, .{ .r32, .rm8 }, .{ 0xf2, 0x0f, 0x38, 0xf0 }, 0, .rex, .crc32 }, + .{ .crc32, .rm, .{ .r32, .rm16 }, .{ 0xf2, 0x0f, 0x38, 0xf1 }, 0, .short, .crc32 }, + .{ .crc32, .rm, .{ .r32, .rm32 }, .{ 0xf2, 0x0f, 0x38, 0xf1 }, 0, .none, .crc32 }, + .{ .crc32, .rm, .{ .r64, .rm8 }, .{ 0xf2, 0x0f, 0x38, 0xf0 }, 0, .long, .crc32 }, + .{ .crc32, .rm, .{ .r64, .rm64 }, .{ 0xf2, 0x0f, 0x38, 0xf1 }, 0, .long, .crc32 }, + + .{ .pcmpgtq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x37 }, 0, .none, .sse4_2 }, + + // ABM + .{ .lzcnt, .rm, .{ .r16, .rm16 }, .{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt }, + .{ .lzcnt, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt }, + .{ .lzcnt, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt }, + + .{ .popcnt, .rm, .{ .r16, .rm16 }, .{ 0xf3, 0x0f, 0xb8 }, 0, .short, .popcnt }, + .{ .popcnt, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt }, + .{ .popcnt, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt }, + + // PCLMUL + .{ .pclmulqdq, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .none, .pclmul }, + + // AES + .{ .aesdec, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xde }, 0, .none, .aes }, + + .{ .aesdeclast, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdf }, 0, .none, .aes }, + + .{ .aesenc, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdc }, 0, .none, .aes }, + + .{ .aesenclast, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdd }, 0, .none, .aes }, + + .{ .aesimc, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdb }, 0, .none, .aes }, + + .{ .aeskeygenassist, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xdf }, 0, .none, .aes }, + + // SHA + .{ .sha1rnds4, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0x3a, 0xcc }, 0, .none, .sha }, + + .{ .sha1nexte, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xc8 }, 0, .none, .sha }, + + .{ .sha1msg1, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xc9 }, 0, .none, .sha }, + + .{ .sha1msg2, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xca }, 0, .none, .sha }, + + .{ .sha256rnds2, .rm0, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xcb }, 0, .none, .sha }, + .{ .sha256rnds2, .rm0, .{ .xmm, .xmm_m128, .xmm0 }, .{ 0x0f, 0x38, 0xcb }, 0, .none, .sha }, + + .{ .sha256msg1, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xcc }, 0, .none, .sha }, + + .{ .sha256msg2, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x38, 0xcd }, 0, .none, .sha }, + + // AVX + .{ .vaddpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx }, + .{ .vaddpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx }, + + .{ .vaddps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x58 }, 0, .vex_128_wig, .avx }, + .{ .vaddps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x58 }, 0, .vex_256_wig, .avx }, + + .{ .vaddsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx }, + + .{ .vaddss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx }, + + .{ .vaddsubpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd0 }, 0, .vex_128_wig, .avx }, + .{ .vaddsubpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd0 }, 0, .vex_256_wig, .avx }, + + .{ .vaddsubps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .vex_128_wig, .avx }, + .{ .vaddsubps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0xf2, 0x0f, 0xd0 }, 0, .vex_256_wig, .avx }, + + .{ .vaesdec, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xde }, 0, .vex_128_wig, .@"aes avx" }, + + .{ .vaesdeclast, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdf }, 0, .vex_128_wig, .@"aes avx" }, + + .{ .vaesenc, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdc }, 0, .vex_128_wig, .@"aes avx" }, + + .{ .vaesenclast, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdd }, 0, .vex_128_wig, .@"aes avx" }, + + .{ .vaesimc, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xdb }, 0, .vex_128_wig, .@"aes avx" }, + + .{ .vaeskeygenassist, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xdf }, 0, .vex_128_wig, .@"aes avx" }, + + .{ .vandnpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x55 }, 0, .vex_128_wig, .avx }, + .{ .vandnpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x55 }, 0, .vex_256_wig, .avx }, + + .{ .vandnps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x55 }, 0, .vex_128_wig, .avx }, + .{ .vandnps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x55 }, 0, .vex_256_wig, .avx }, + + .{ .vandpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x54 }, 0, .vex_128_wig, .avx }, + .{ .vandpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x54 }, 0, .vex_256_wig, .avx }, + + .{ .vandps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x54 }, 0, .vex_128_wig, .avx }, + .{ .vandps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x54 }, 0, .vex_256_wig, .avx }, + + .{ .vblendpd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .vex_128_wig, .avx }, + .{ .vblendpd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .vex_256_wig, .avx }, + + .{ .vblendps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .vex_128_wig, .avx }, + .{ .vblendps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .vex_256_wig, .avx }, + + .{ .vblendvpd, .rvmr, .{ .xmm, .xmm, .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x3a, 0x4b }, 0, .vex_128_w0, .avx }, + .{ .vblendvpd, .rvmr, .{ .ymm, .ymm, .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x3a, 0x4b }, 0, .vex_256_w0, .avx }, + + .{ .vblendvps, .rvmr, .{ .xmm, .xmm, .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x3a, 0x4a }, 0, .vex_128_w0, .avx }, + .{ .vblendvps, .rvmr, .{ .ymm, .ymm, .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x3a, 0x4a }, 0, .vex_256_w0, .avx }, + + .{ .vbroadcastss, .rm, .{ .xmm, .m32 }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx }, + .{ .vbroadcastss, .rm, .{ .ymm, .m32 }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx }, + .{ .vbroadcastsd, .rm, .{ .ymm, .m64 }, .{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx }, + .{ .vbroadcastf128, .rm, .{ .ymm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x1a }, 0, .vex_256_w0, .avx }, + + .{ .vcmppd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc2 }, 0, .vex_128_wig, .avx }, + .{ .vcmppd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0xc2 }, 0, .vex_256_wig, .avx }, + + .{ .vcmpps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc2 }, 0, .vex_128_wig, .avx }, + .{ .vcmpps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x0f, 0xc2 }, 0, .vex_256_wig, .avx }, + + .{ .vcmpsd, .rvmi, .{ .xmm, .xmm, .xmm_m64, .imm8 }, .{ 0xf2, 0x0f, 0xc2 }, 0, .vex_lig_wig, .avx }, + + .{ .vcmpss, .rvmi, .{ .xmm, .xmm, .xmm_m32, .imm8 }, .{ 0xf3, 0x0f, 0xc2 }, 0, .vex_lig_wig, .avx }, + + .{ .vcomisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2f }, 0, .vex_lig_wig, .avx }, + + .{ .vcomiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2f }, 0, .vex_lig_wig, .avx }, + + .{ .vcvtdq2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx }, + .{ .vcvtdq2pd, .rm, .{ .ymm, .xmm_m128 }, .{ 0xf3, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx }, + + .{ .vcvtdq2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x5b }, 0, .vex_128_wig, .avx }, + .{ .vcvtdq2ps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x5b }, 0, .vex_256_wig, .avx }, + + .{ .vcvtpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx }, + .{ .vcvtpd2dq, .rm, .{ .xmm, .ymm_m256 }, .{ 0xf2, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx }, + + .{ .vcvtpd2ps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5a }, 0, .vex_128_wig, .avx }, + .{ .vcvtpd2ps, .rm, .{ .xmm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5a }, 0, .vex_256_wig, .avx }, + + .{ .vcvtps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5b }, 0, .vex_128_wig, .avx }, + .{ .vcvtps2dq, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5b }, 0, .vex_256_wig, .avx }, + + .{ .vcvtps2pd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x0f, 0x5a }, 0, .vex_128_wig, .avx }, + .{ .vcvtps2pd, .rm, .{ .ymm, .xmm_m128 }, .{ 0x0f, 0x5a }, 0, .vex_256_wig, .avx }, + + .{ .vcvtsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .vex_lig_w0, .sse2 }, + .{ .vcvtsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2d }, 0, .vex_lig_w1, .sse2 }, + + .{ .vcvtsd2ss, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx }, + + .{ .vcvtsi2sd, .rvm, .{ .xmm, .xmm, .rm32 }, .{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx }, + .{ .vcvtsi2sd, .rvm, .{ .xmm, .xmm, .rm64 }, .{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx }, + + .{ .vcvtsi2ss, .rvm, .{ .xmm, .xmm, .rm32 }, .{ 0xf3, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx }, + .{ .vcvtsi2ss, .rvm, .{ .xmm, .xmm, .rm64 }, .{ 0xf3, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx }, + + .{ .vcvtss2sd, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx }, + + .{ .vcvtss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .vex_lig_w0, .avx }, + .{ .vcvtss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2d }, 0, .vex_lig_w1, .avx }, + + .{ .vcvttpd2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx }, + .{ .vcvttpd2dq, .rm, .{ .xmm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx }, + + .{ .vcvttps2dq, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x5b }, 0, .vex_128_wig, .avx }, + .{ .vcvttps2dq, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x5b }, 0, .vex_256_wig, .avx }, + + .{ .vcvttsd2si, .rm, .{ .r32, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .vex_lig_w0, .sse2 }, + .{ .vcvttsd2si, .rm, .{ .r64, .xmm_m64 }, .{ 0xf2, 0x0f, 0x2c }, 0, .vex_lig_w1, .sse2 }, + + .{ .vcvttss2si, .rm, .{ .r32, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .vex_lig_w0, .avx }, + .{ .vcvttss2si, .rm, .{ .r64, .xmm_m32 }, .{ 0xf3, 0x0f, 0x2c }, 0, .vex_lig_w1, .avx }, + + .{ .vdppd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x41 }, 0, .vex_128_wig, .avx }, + + .{ .vdpps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x40 }, 0, .vex_128_wig, .avx }, + .{ .vdpps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x40 }, 0, .vex_256_wig, .avx }, + + .{ .vdivpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5e }, 0, .vex_128_wig, .avx }, + .{ .vdivpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5e }, 0, .vex_256_wig, .avx }, + + .{ .vdivps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5e }, 0, .vex_128_wig, .avx }, + .{ .vdivps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5e }, 0, .vex_256_wig, .avx }, + + .{ .vdivsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx }, + + .{ .vdivss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx }, + + .{ .vextractf128, .mri, .{ .xmm_m128, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x19 }, 0, .vex_256_w0, .avx }, + + .{ .vextractps, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .vex_128_wig, .avx }, + + .{ .vgf2p8affineinvqb, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xcf }, 0, .vex_128_w1, .@"gfni avx" }, + .{ .vgf2p8affineinvqb, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xcf }, 0, .vex_256_w1, .@"gfni avx" }, + + .{ .vgf2p8affineqb, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xce }, 0, .vex_128_w1, .@"gfni avx" }, + .{ .vgf2p8affineqb, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0xce }, 0, .vex_256_w1, .@"gfni avx" }, + + .{ .vgf2p8mulb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xcf }, 0, .vex_128_w0, .@"gfni avx" }, + .{ .vgf2p8mulb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xcf }, 0, .vex_256_w0, .@"gfni avx" }, + + .{ .vhaddpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x7c }, 0, .vex_128_wig, .avx }, + .{ .vhaddpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x7c }, 0, .vex_256_wig, .avx }, + + .{ .vhaddps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0xf2, 0x0f, 0x7c }, 0, .vex_128_wig, .avx }, + .{ .vhaddps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0xf2, 0x0f, 0x7c }, 0, .vex_256_wig, .avx }, + + .{ .vinsertf128, .rvmi, .{ .ymm, .ymm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x18 }, 0, .vex_256_w0, .avx }, + + .{ .vinsertps, .rvmi, .{ .xmm, .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .vex_128_wig, .avx }, + + .{ .vlddqu, .rm, .{ .xmm, .m128 }, .{ 0xf2, 0x0f, 0xf0 }, 0, .vex_128_wig, .avx }, + .{ .vlddqu, .rm, .{ .ymm, .m256 }, .{ 0xf2, 0x0f, 0xf0 }, 0, .vex_256_wig, .avx }, + + .{ .vldmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 2, .vex_lz_wig, .avx }, + + .{ .vmaskmovps, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x2c }, 0, .vex_128_w0, .avx }, + .{ .vmaskmovps, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x2c }, 0, .vex_256_w0, .avx }, + .{ .vmaskmovpd, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x2d }, 0, .vex_128_w0, .avx }, + .{ .vmaskmovpd, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x2d }, 0, .vex_256_w0, .avx }, + .{ .vmaskmovps, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x2e }, 0, .vex_128_w0, .avx }, + .{ .vmaskmovps, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x2e }, 0, .vex_256_w0, .avx }, + .{ .vmaskmovpd, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x2f }, 0, .vex_128_w0, .avx }, + .{ .vmaskmovpd, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x2f }, 0, .vex_256_w0, .avx }, + + .{ .vmaxpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5f }, 0, .vex_128_wig, .avx }, + .{ .vmaxpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5f }, 0, .vex_256_wig, .avx }, + + .{ .vmaxps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5f }, 0, .vex_128_wig, .avx }, + .{ .vmaxps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5f }, 0, .vex_256_wig, .avx }, + + .{ .vmaxsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx }, + + .{ .vmaxss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx }, + + .{ .vmovmskps, .rm, .{ .r32, .xmm }, .{ 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, + .{ .vmovmskps, .rm, .{ .r64, .xmm }, .{ 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, + .{ .vmovmskps, .rm, .{ .r32, .ymm }, .{ 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, + .{ .vmovmskps, .rm, .{ .r64, .ymm }, .{ 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, + + .{ .vmovmskpd, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, + .{ .vmovmskpd, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_128_wig, .avx }, + .{ .vmovmskpd, .rm, .{ .r32, .ymm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, + .{ .vmovmskpd, .rm, .{ .r64, .ymm }, .{ 0x66, 0x0f, 0x50 }, 0, .vex_256_wig, .avx }, + + .{ .vminpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5d }, 0, .vex_128_wig, .avx }, + .{ .vminpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5d }, 0, .vex_256_wig, .avx }, + + .{ .vminps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5d }, 0, .vex_128_wig, .avx }, + .{ .vminps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5d }, 0, .vex_256_wig, .avx }, + + .{ .vminsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx }, + + .{ .vminss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx }, + + .{ .vmovapd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x28 }, 0, .vex_128_wig, .avx }, + .{ .vmovapd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x29 }, 0, .vex_128_wig, .avx }, + .{ .vmovapd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x28 }, 0, .vex_256_wig, .avx }, + .{ .vmovapd, .mr, .{ .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x29 }, 0, .vex_256_wig, .avx }, + + .{ .vmovaps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x28 }, 0, .vex_128_wig, .avx }, + .{ .vmovaps, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x29 }, 0, .vex_128_wig, .avx }, + .{ .vmovaps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x28 }, 0, .vex_256_wig, .avx }, + .{ .vmovaps, .mr, .{ .ymm_m256, .ymm }, .{ 0x0f, 0x29 }, 0, .vex_256_wig, .avx }, + + .{ .vmovd, .rm, .{ .xmm, .rm32 }, .{ 0x66, 0x0f, 0x6e }, 0, .vex_128_w0, .avx }, + .{ .vmovq, .rm, .{ .xmm, .rm64 }, .{ 0x66, 0x0f, 0x6e }, 0, .vex_128_w1, .avx }, + .{ .vmovd, .mr, .{ .rm32, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .vex_128_w0, .avx }, + .{ .vmovq, .mr, .{ .rm64, .xmm }, .{ 0x66, 0x0f, 0x7e }, 0, .vex_128_w1, .avx }, + + .{ .vmovddup, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, + .{ .vmovddup, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx }, + + .{ .vmovdqa, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6f }, 0, .vex_128_wig, .avx }, + .{ .vmovdqa, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x7f }, 0, .vex_128_wig, .avx }, + .{ .vmovdqa, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6f }, 0, .vex_256_wig, .avx }, + .{ .vmovdqa, .mr, .{ .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x7f }, 0, .vex_256_wig, .avx }, + + .{ .vmovdqu, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x6f }, 0, .vex_128_wig, .avx }, + .{ .vmovdqu, .mr, .{ .xmm_m128, .xmm }, .{ 0xf3, 0x0f, 0x7f }, 0, .vex_128_wig, .avx }, + .{ .vmovdqu, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x6f }, 0, .vex_256_wig, .avx }, + .{ .vmovdqu, .mr, .{ .ymm_m256, .ymm }, .{ 0xf3, 0x0f, 0x7f }, 0, .vex_256_wig, .avx }, + + .{ .vmovhlps, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, + + .{ .vmovhpd, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x66, 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, + .{ .vmovhpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x17 }, 0, .vex_128_wig, .avx }, + + .{ .vmovhps, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, + .{ .vmovhps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x17 }, 0, .vex_128_wig, .avx }, + + .{ .vmovlhps, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, + + .{ .vmovlpd, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x66, 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, + .{ .vmovlpd, .mr, .{ .m64, .xmm }, .{ 0x66, 0x0f, 0x13 }, 0, .vex_128_wig, .avx }, + + .{ .vmovlps, .rvm, .{ .xmm, .xmm, .m64 }, .{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, + .{ .vmovlps, .mr, .{ .m64, .xmm }, .{ 0x0f, 0x13 }, 0, .vex_128_wig, .avx }, + + .{ .vmovq, .rm, .{ .xmm, .xmm_m64 }, .{ 0xf3, 0x0f, 0x7e }, 0, .vex_128_wig, .avx }, + .{ .vmovq, .mr, .{ .xmm_m64, .xmm }, .{ 0x66, 0x0f, 0xd6 }, 0, .vex_128_wig, .avx }, + + .{ .vmovsd, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, + .{ .vmovsd, .rm, .{ .xmm, .m64 }, .{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, + .{ .vmovsd, .mvr, .{ .xmm, .xmm, .xmm }, .{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, + .{ .vmovsd, .mr, .{ .m64, .xmm }, .{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, + + .{ .vmovshdup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x16 }, 0, .vex_128_wig, .avx }, + .{ .vmovshdup, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x16 }, 0, .vex_256_wig, .avx }, + + .{ .vmovsldup, .rm, .{ .xmm, .xmm_m128 }, .{ 0xf3, 0x0f, 0x12 }, 0, .vex_128_wig, .avx }, + .{ .vmovsldup, .rm, .{ .ymm, .ymm_m256 }, .{ 0xf3, 0x0f, 0x12 }, 0, .vex_256_wig, .avx }, + + .{ .vmovss, .rvm, .{ .xmm, .xmm, .xmm }, .{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, + .{ .vmovss, .rm, .{ .xmm, .m32 }, .{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx }, + .{ .vmovss, .mvr, .{ .xmm, .xmm, .xmm }, .{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, + .{ .vmovss, .mr, .{ .m32, .xmm }, .{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx }, + + .{ .vmovupd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x10 }, 0, .vex_128_wig, .avx }, + .{ .vmovupd, .mr, .{ .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x11 }, 0, .vex_128_wig, .avx }, + .{ .vmovupd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x10 }, 0, .vex_256_wig, .avx }, + .{ .vmovupd, .mr, .{ .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x11 }, 0, .vex_256_wig, .avx }, + + .{ .vmovups, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x10 }, 0, .vex_128_wig, .avx }, + .{ .vmovups, .mr, .{ .xmm_m128, .xmm }, .{ 0x0f, 0x11 }, 0, .vex_128_wig, .avx }, + .{ .vmovups, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x10 }, 0, .vex_256_wig, .avx }, + .{ .vmovups, .mr, .{ .ymm_m256, .ymm }, .{ 0x0f, 0x11 }, 0, .vex_256_wig, .avx }, + + .{ .vmulpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x59 }, 0, .vex_128_wig, .avx }, + .{ .vmulpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x59 }, 0, .vex_256_wig, .avx }, + + .{ .vmulps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x59 }, 0, .vex_128_wig, .avx }, + .{ .vmulps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x59 }, 0, .vex_256_wig, .avx }, + + .{ .vmulsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx }, + + .{ .vmulss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx }, + + .{ .vorpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x56 }, 0, .vex_128_wig, .avx }, + .{ .vorpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x56 }, 0, .vex_256_wig, .avx }, + + .{ .vorps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx }, + .{ .vorps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx }, + + .{ .vpabsb, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_128_wig, .avx }, + .{ .vpabsd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_128_wig, .avx }, + .{ .vpabsw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_128_wig, .avx }, + + .{ .vpacksswb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x63 }, 0, .vex_128_wig, .avx }, + .{ .vpackssdw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6b }, 0, .vex_128_wig, .avx }, + + .{ .vpackusdw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_128_wig, .avx }, + + .{ .vpackuswb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x67 }, 0, .vex_128_wig, .avx }, + + .{ .vpaddb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfc }, 0, .vex_128_wig, .avx }, + .{ .vpaddw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfd }, 0, .vex_128_wig, .avx }, + .{ .vpaddd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfe }, 0, .vex_128_wig, .avx }, + .{ .vpaddq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd4 }, 0, .vex_128_wig, .avx }, + + .{ .vpaddsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xec }, 0, .vex_128_wig, .avx }, + .{ .vpaddsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xed }, 0, .vex_128_wig, .avx }, + + .{ .vpaddusb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdc }, 0, .vex_128_wig, .avx }, + .{ .vpaddusw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdd }, 0, .vex_128_wig, .avx }, + + .{ .vpalignr, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .vex_128_wig, .avx }, + + .{ .vpand, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdb }, 0, .vex_128_wig, .avx }, + + .{ .vpandn, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx }, + + .{ .vpblendvb, .rvmr, .{ .xmm, .xmm, .xmm_m128, .xmm }, .{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_128_w0, .avx }, + + .{ .vpblendw, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_128_wig, .avx }, + + .{ .vpclmulqdq, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" }, + + .{ .vpcmpeqb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx }, + .{ .vpcmpeqw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x75 }, 0, .vex_128_wig, .avx }, + .{ .vpcmpeqd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x76 }, 0, .vex_128_wig, .avx }, + + .{ .vpcmpeqq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x29 }, 0, .vex_128_wig, .avx }, + + .{ .vpcmpgtb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x64 }, 0, .vex_128_wig, .avx }, + .{ .vpcmpgtw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x65 }, 0, .vex_128_wig, .avx }, + .{ .vpcmpgtd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x66 }, 0, .vex_128_wig, .avx }, + + .{ .vpcmpgtq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x37 }, 0, .vex_128_wig, .avx }, + + .{ .vperm2f128, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x06 }, 0, .vex_256_w0, .avx }, + + .{ .vpermilpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0d }, 0, .vex_128_w0, .avx }, + .{ .vpermilpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0d }, 0, .vex_256_w0, .avx }, + .{ .vpermilpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x05 }, 0, .vex_128_w0, .avx }, + .{ .vpermilpd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x05 }, 0, .vex_256_w0, .avx }, + + .{ .vpermilpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0c }, 0, .vex_128_w0, .avx }, + .{ .vpermilps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x04 }, 0, .vex_128_w0, .avx }, + .{ .vpermilps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0c }, 0, .vex_256_w0, .avx }, + .{ .vpermilps, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x04 }, 0, .vex_256_w0, .avx }, + + .{ .vpextrb, .mri, .{ .r32_m8, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx }, + .{ .vpextrd, .mri, .{ .rm32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx }, + .{ .vpextrq, .mri, .{ .rm64, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx }, + + .{ .vpextrw, .rmi, .{ .r32, .xmm, .imm8 }, .{ 0x66, 0x0f, 0xc5 }, 0, .vex_128_w0, .avx }, + .{ .vpextrw, .mri, .{ .r32_m16, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_w0, .avx }, + + .{ .vphaddw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x01 }, 0, .vex_128_wig, .avx }, + .{ .vphaddd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x02 }, 0, .vex_128_wig, .avx }, + + .{ .vphaddsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x03 }, 0, .vex_128_wig, .avx }, + + .{ .vphminposuw, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x41 }, 0, .vex_128_wig, .avx }, + + .{ .vphsubw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x05 }, 0, .vex_128_wig, .avx }, + .{ .vphsubd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x06 }, 0, .vex_128_wig, .avx }, + + .{ .vphsubsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x07 }, 0, .vex_128_wig, .avx }, + + .{ .vpinsrb, .rvmi, .{ .xmm, .xmm, .r32_m8, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx }, + .{ .vpinsrd, .rvmi, .{ .xmm, .xmm, .rm32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx }, + .{ .vpinsrq, .rvmi, .{ .xmm, .xmm, .rm64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx }, + + .{ .vpinsrw, .rvmi, .{ .xmm, .xmm, .r32_m16, .imm8 }, .{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_w0, .avx }, + + .{ .vpmaddubsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x04 }, 0, .vex_128_wig, .avx }, + + .{ .vpmaddwd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf5 }, 0, .vex_128_wig, .avx }, + + .{ .vpmaxsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_128_wig, .avx }, + .{ .vpmaxsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xee }, 0, .vex_128_wig, .avx }, + .{ .vpmaxsd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3d }, 0, .vex_128_wig, .avx }, + + .{ .vpmaxub, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xde }, 0, .vex_128_wig, .avx }, + .{ .vpmaxuw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3e }, 0, .vex_128_wig, .avx }, + + .{ .vpmaxud, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3f }, 0, .vex_128_wig, .avx }, + + .{ .vpminsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x38 }, 0, .vex_128_wig, .avx }, + .{ .vpminsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xea }, 0, .vex_128_wig, .avx }, + .{ .vpminsd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x39 }, 0, .vex_128_wig, .avx }, + + .{ .vpminub, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xda }, 0, .vex_128_wig, .avx }, + .{ .vpminuw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3a }, 0, .vex_128_wig, .avx }, + + .{ .vpminud, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x3b }, 0, .vex_128_wig, .avx }, + + .{ .vpmovmskb, .rm, .{ .r32, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx }, + .{ .vpmovmskb, .rm, .{ .r64, .xmm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx }, + + .{ .vpmovsxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_128_wig, .avx }, + .{ .vpmovsxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_128_wig, .avx }, + .{ .vpmovsxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_128_wig, .avx }, + .{ .vpmovsxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_128_wig, .avx }, + .{ .vpmovsxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_128_wig, .avx }, + .{ .vpmovsxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_128_wig, .avx }, + + .{ .vpmovzxbw, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_128_wig, .avx }, + .{ .vpmovzxbd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_128_wig, .avx }, + .{ .vpmovzxbq, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_128_wig, .avx }, + .{ .vpmovzxwd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_128_wig, .avx }, + .{ .vpmovzxwq, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx }, + .{ .vpmovzxdq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx }, + + .{ .vpmuldq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .vex_128_wig, .avx }, + + .{ .vpmulhrsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0b }, 0, .vex_128_wig, .avx }, + + .{ .vpmulhuw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe4 }, 0, .vex_128_wig, .avx }, + + .{ .vpmulhw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx }, + + .{ .vpmulld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx }, + + .{ .vpmullw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd5 }, 0, .vex_128_wig, .avx }, + + .{ .vpmuludq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf4 }, 0, .vex_128_wig, .avx }, + + .{ .vpor, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx }, + + .{ .vpsadbw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf6 }, 0, .vex_128_wig, .avx }, + + .{ .vpshufb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx }, + + .{ .vpshufd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx }, + + .{ .vpshufhw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx }, + + .{ .vpshuflw, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0xf2, 0x0f, 0x70 }, 0, .vex_128_wig, .avx }, + + .{ .vpsignb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x08 }, 0, .vex_128_wig, .avx }, + .{ .vpsignw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x09 }, 0, .vex_128_wig, .avx }, + .{ .vpsignd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0a }, 0, .vex_128_wig, .avx }, + + .{ .vpsllw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf1 }, 0, .vex_128_wig, .avx }, + .{ .vpsllw, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 6, .vex_128_wig, .avx }, + .{ .vpslld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf2 }, 0, .vex_128_wig, .avx }, + .{ .vpslld, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 6, .vex_128_wig, .avx }, + .{ .vpsllq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf3 }, 0, .vex_128_wig, .avx }, + .{ .vpsllq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 6, .vex_128_wig, .avx }, + + .{ .vpslldq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 7, .vex_128_wig, .avx }, + + .{ .vpsraw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe1 }, 0, .vex_128_wig, .avx }, + .{ .vpsraw, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 4, .vex_128_wig, .avx }, + .{ .vpsrad, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe2 }, 0, .vex_128_wig, .avx }, + .{ .vpsrad, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 4, .vex_128_wig, .avx }, + + .{ .vpsrlw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx }, + .{ .vpsrlw, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 2, .vex_128_wig, .avx }, + .{ .vpsrld, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd2 }, 0, .vex_128_wig, .avx }, + .{ .vpsrld, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 2, .vex_128_wig, .avx }, + .{ .vpsrlq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd3 }, 0, .vex_128_wig, .avx }, + .{ .vpsrlq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 2, .vex_128_wig, .avx }, + + .{ .vpsrldq, .vmi, .{ .xmm, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 3, .vex_128_wig, .avx }, + + .{ .vpsubb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf8 }, 0, .vex_128_wig, .avx }, + .{ .vpsubw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf9 }, 0, .vex_128_wig, .avx }, + .{ .vpsubd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfa }, 0, .vex_128_wig, .avx }, + + .{ .vpsubsb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe8 }, 0, .vex_128_wig, .avx }, + .{ .vpsubsw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe9 }, 0, .vex_128_wig, .avx }, + + .{ .vpsubq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xfb }, 0, .vex_128_wig, .avx }, + + .{ .vpsubusb, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd8 }, 0, .vex_128_wig, .avx }, + .{ .vpsubusw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd9 }, 0, .vex_128_wig, .avx }, + + .{ .vptest, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_128_wig, .avx }, + .{ .vptest, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x17 }, 0, .vex_256_wig, .avx }, + + .{ .vpunpckhbw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx }, + .{ .vpunpckhwd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx }, + .{ .vpunpckhdq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx }, + .{ .vpunpckhqdq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6d }, 0, .vex_128_wig, .avx }, + + .{ .vpunpcklbw, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x60 }, 0, .vex_128_wig, .avx }, + .{ .vpunpcklwd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x61 }, 0, .vex_128_wig, .avx }, + .{ .vpunpckldq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx }, + .{ .vpunpcklqdq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx }, + + .{ .vpxor, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0xef }, 0, .vex_128_wig, .avx }, + + .{ .vroundpd, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_128_wig, .avx }, + .{ .vroundpd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_256_wig, .avx }, + + .{ .vroundps, .rmi, .{ .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_128_wig, .avx }, + .{ .vroundps, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_256_wig, .avx }, + + .{ .vroundsd, .rvmi, .{ .xmm, .xmm, .xmm_m64, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .vex_lig_wig, .avx }, + + .{ .vroundss, .rvmi, .{ .xmm, .xmm, .xmm_m32, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .vex_lig_wig, .avx }, + + .{ .vshufpd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0xc6 }, 0, .vex_128_wig, .avx }, + .{ .vshufpd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0xc6 }, 0, .vex_256_wig, .avx }, + + .{ .vshufps, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x0f, 0xc6 }, 0, .vex_128_wig, .avx }, + .{ .vshufps, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x0f, 0xc6 }, 0, .vex_256_wig, .avx }, + + .{ .vsqrtpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx }, + .{ .vsqrtpd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx }, + + .{ .vsqrtps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x0f, 0x51 }, 0, .vex_128_wig, .avx }, + .{ .vsqrtps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x0f, 0x51 }, 0, .vex_256_wig, .avx }, + + .{ .vsqrtsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx }, + + .{ .vsqrtss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx }, + + .{ .vstmxcsr, .m, .{ .m32 }, .{ 0x0f, 0xae }, 3, .vex_lz_wig, .avx }, + + .{ .vsubpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x5c }, 0, .vex_128_wig, .avx }, + .{ .vsubpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x5c }, 0, .vex_256_wig, .avx }, + + .{ .vsubps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x5c }, 0, .vex_128_wig, .avx }, + .{ .vsubps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x5c }, 0, .vex_256_wig, .avx }, + + .{ .vsubsd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0xf2, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx }, + + .{ .vsubss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx }, + + .{ .vtestps, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_128_w0, .avx }, + .{ .vtestps, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0e }, 0, .vex_256_w0, .avx }, + .{ .vtestpd, .rm, .{ .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_128_w0, .avx }, + .{ .vtestpd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_256_w0, .avx }, + + .{ .vucomisd, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x2e }, 0, .vex_lig_wig, .avx }, + + .{ .vucomiss, .rm, .{ .xmm, .xmm_m32 }, .{ 0x0f, 0x2e }, 0, .vex_lig_wig, .avx }, + + .{ .vunpckhpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx }, + .{ .vunpckhpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x15 }, 0, .vex_256_wig, .avx }, + + .{ .vunpckhps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x15 }, 0, .vex_128_wig, .avx }, + .{ .vunpckhps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x15 }, 0, .vex_256_wig, .avx }, + + .{ .vunpcklpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x14 }, 0, .vex_128_wig, .avx }, + .{ .vunpcklpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x14 }, 0, .vex_256_wig, .avx }, + + .{ .vunpcklps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x14 }, 0, .vex_128_wig, .avx }, + .{ .vunpcklps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x14 }, 0, .vex_256_wig, .avx }, + + .{ .vxorpd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx }, + .{ .vxorpd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx }, + + .{ .vxorps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x0f, 0x57 }, 0, .vex_128_wig, .avx }, + .{ .vxorps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x0f, 0x57 }, 0, .vex_256_wig, .avx }, + + // BMI + .{ .andn, .rvm, .{ .r32, .r32, .rm32 }, .{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w0, .bmi }, + .{ .andn, .rvm, .{ .r64, .r64, .rm64 }, .{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w1, .bmi }, + + .{ .bextr, .rmv, .{ .r32, .rm32, .r32 }, .{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi }, + .{ .bextr, .rmv, .{ .r64, .rm64, .r64 }, .{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi }, + + .{ .blsi, .vm, .{ .r32, .rm32 }, .{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w0, .bmi }, + .{ .blsi, .vm, .{ .r64, .rm64 }, .{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w1, .bmi }, + + .{ .blsmsk, .vm, .{ .r32, .rm32 }, .{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w0, .bmi }, + .{ .blsmsk, .vm, .{ .r64, .rm64 }, .{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w1, .bmi }, + + .{ .blsr, .vm, .{ .r32, .rm32 }, .{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w0, .bmi }, + .{ .blsr, .vm, .{ .r64, .rm64 }, .{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w1, .bmi }, + + .{ .tzcnt, .rm, .{ .r16, .rm16 }, .{ 0xf3, 0x0f, 0xbc }, 0, .short, .bmi }, + .{ .tzcnt, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi }, + .{ .tzcnt, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi }, + + // BMI2 + .{ .bzhi, .rmv, .{ .r32, .rm32, .r32 }, .{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, + .{ .bzhi, .rmv, .{ .r64, .rm64, .r64 }, .{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, + + .{ .mulx, .rvm, .{ .r32, .r32, .rm32 }, .{ 0xf2, 0x0f, 0x38, 0xf6 }, 0, .vex_lz_w0, .bmi2 }, + .{ .mulx, .rvm, .{ .r64, .r64, .rm64 }, .{ 0xf2, 0x0f, 0x38, 0xf6 }, 0, .vex_lz_w1, .bmi2 }, + + .{ .pdep, .rvm, .{ .r32, .r32, .rm32 }, .{ 0xf2, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, + .{ .pdep, .rvm, .{ .r64, .r64, .rm64 }, .{ 0xf2, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, + + .{ .pext, .rvm, .{ .r32, .r32, .rm32 }, .{ 0xf3, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, + .{ .pext, .rvm, .{ .r64, .r64, .rm64 }, .{ 0xf3, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, + + .{ .rorx, .rmi, .{ .r32, .rm32, .imm8 }, .{ 0xf2, 0x0f, 0x3a, 0xf0 }, 0, .vex_lz_w0, .bmi2 }, + .{ .rorx, .rmi, .{ .r64, .rm64, .imm8 }, .{ 0xf2, 0x0f, 0x3a, 0xf0 }, 0, .vex_lz_w1, .bmi2 }, + + .{ .sarx, .rmv, .{ .r32, .rm32, .r32 }, .{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, + .{ .shlx, .rmv, .{ .r32, .rm32, .r32 }, .{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, + .{ .shrx, .rmv, .{ .r32, .rm32, .r32 }, .{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, + .{ .sarx, .rmv, .{ .r64, .rm64, .r64 }, .{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, + .{ .shlx, .rmv, .{ .r64, .rm64, .r64 }, .{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, + .{ .shrx, .rmv, .{ .r64, .rm64, .r64 }, .{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, + + // F16C + .{ .vcvtph2ps, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c }, + .{ .vcvtph2ps, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c }, + + .{ .vcvtps2ph, .mri, .{ .xmm_m64, .xmm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128_w0, .f16c }, + .{ .vcvtps2ph, .mri, .{ .xmm_m128, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_256_w0, .f16c }, + + // FMA + .{ .vfmadd132pd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w1, .fma }, + .{ .vfmadd213pd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w1, .fma }, + .{ .vfmadd231pd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w1, .fma }, + .{ .vfmadd132pd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w1, .fma }, + .{ .vfmadd213pd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w1, .fma }, + .{ .vfmadd231pd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w1, .fma }, + + .{ .vfmadd132ps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w0, .fma }, + .{ .vfmadd213ps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w0, .fma }, + .{ .vfmadd231ps, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w0, .fma }, + .{ .vfmadd132ps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w0, .fma }, + .{ .vfmadd213ps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w0, .fma }, + .{ .vfmadd231ps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w0, .fma }, + + .{ .vfmadd132sd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w1, .fma }, + .{ .vfmadd213sd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w1, .fma }, + .{ .vfmadd231sd, .rvm, .{ .xmm, .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w1, .fma }, + + .{ .vfmadd132ss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w0, .fma }, + .{ .vfmadd213ss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w0, .fma }, + .{ .vfmadd231ss, .rvm, .{ .xmm, .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w0, .fma }, + + // VPCLMULQDQ + .{ .vpclmulqdq, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_256_wig, .vpclmulqdq }, + + // AVX2 + .{ .vbroadcastss, .rm, .{ .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx2 }, + .{ .vbroadcastss, .rm, .{ .ymm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 }, + .{ .vbroadcastsd, .rm, .{ .ymm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 }, + + .{ .vextracti128, .mri, .{ .xmm_m128, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 }, + + .{ .vinserti128, .rvmi, .{ .ymm, .ymm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 }, + + .{ .vpabsb, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 }, + .{ .vpabsd, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 }, + .{ .vpabsw, .rm, .{ .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 }, + + .{ .vpacksswb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x63 }, 0, .vex_256_wig, .avx2 }, + .{ .vpackssdw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6b }, 0, .vex_256_wig, .avx2 }, + + .{ .vpackusdw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_256_wig, .avx2 }, + + .{ .vpackuswb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x67 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpaddb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfc }, 0, .vex_256_wig, .avx2 }, + .{ .vpaddw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfd }, 0, .vex_256_wig, .avx2 }, + .{ .vpaddd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfe }, 0, .vex_256_wig, .avx2 }, + .{ .vpaddq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd4 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpaddsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xec }, 0, .vex_256_wig, .avx2 }, + .{ .vpaddsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xed }, 0, .vex_256_wig, .avx2 }, + + .{ .vpaddusb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdc }, 0, .vex_256_wig, .avx2 }, + .{ .vpaddusw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdd }, 0, .vex_256_wig, .avx2 }, + + .{ .vpalignr, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .vex_256_wig, .avx2 }, + + .{ .vpand, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdb }, 0, .vex_256_wig, .avx2 }, + + .{ .vpandn, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 }, + + .{ .vpblendd, .rvmi, .{ .xmm, .xmm, .xmm_m128, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_128_w0, .avx2 }, + .{ .vpblendd, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_256_w0, .avx2 }, + + .{ .vpblendvb, .rvmr, .{ .ymm, .ymm, .ymm_m256, .ymm }, .{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_256_w0, .avx2 }, + + .{ .vpblendw, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_256_wig, .avx2 }, + + .{ .vpbroadcastb, .rm, .{ .xmm, .xmm_m8 }, .{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 }, + .{ .vpbroadcastb, .rm, .{ .ymm, .xmm_m8 }, .{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 }, + .{ .vpbroadcastw, .rm, .{ .xmm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 }, + .{ .vpbroadcastw, .rm, .{ .ymm, .xmm_m16 }, .{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_256_w0, .avx2 }, + .{ .vpbroadcastd, .rm, .{ .xmm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_128_w0, .avx2 }, + .{ .vpbroadcastd, .rm, .{ .ymm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 }, + .{ .vpbroadcastq, .rm, .{ .xmm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 }, + .{ .vpbroadcastq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 }, + .{ .vbroadcasti128, .rm, .{ .ymm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 }, + + .{ .vpcmpeqb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 }, + .{ .vpcmpeqw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 }, + .{ .vpcmpeqd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpcmpeqq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x29 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpcmpgtb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x64 }, 0, .vex_256_wig, .avx2 }, + .{ .vpcmpgtw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x65 }, 0, .vex_256_wig, .avx2 }, + .{ .vpcmpgtd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x66 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpcmpgtq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x37 }, 0, .vex_256_wig, .avx2 }, + + .{ .vphaddw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x01 }, 0, .vex_256_wig, .avx2 }, + .{ .vphaddd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x02 }, 0, .vex_256_wig, .avx2 }, + + .{ .vphaddsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x03 }, 0, .vex_256_wig, .avx2 }, + + .{ .vphaddw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x05 }, 0, .vex_256_wig, .avx2 }, + .{ .vphaddd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x06 }, 0, .vex_256_wig, .avx2 }, + + .{ .vphaddsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x07 }, 0, .vex_256_wig, .avx2 }, + + .{ .vperm2i128, .rvmi, .{ .ymm, .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x46 }, 0, .vex_256_w0, .avx2 }, + + .{ .vpermd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x36 }, 0, .vex_256_w0, .avx2 }, + + .{ .vpermpd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x01 }, 0, .vex_256_w1, .avx2 }, + + .{ .vpermps, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x16 }, 0, .vex_256_w0, .avx2 }, + + .{ .vpermq, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x3a, 0x00 }, 0, .vex_256_w1, .avx2 }, + + .{ .vpmaddubsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x04 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmaddwd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf5 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmaskmovd, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_128_w0, .avx2 }, + .{ .vpmaskmovd, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_256_w0, .avx2 }, + .{ .vpmaskmovq, .rvm, .{ .xmm, .xmm, .m128 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_128_w1, .avx2 }, + .{ .vpmaskmovq, .rvm, .{ .ymm, .ymm, .m256 }, .{ 0x66, 0x0f, 0x38, 0x8c }, 0, .vex_256_w1, .avx2 }, + .{ .vpmaskmovd, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_128_w0, .avx2 }, + .{ .vpmaskmovd, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_256_w0, .avx2 }, + .{ .vpmaskmovq, .mvr, .{ .m128, .xmm, .xmm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_128_w1, .avx2 }, + .{ .vpmaskmovq, .mvr, .{ .m256, .ymm, .ymm }, .{ 0x66, 0x0f, 0x38, 0x8e }, 0, .vex_256_w1, .avx2 }, + + .{ .vpmaxsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3c }, 0, .vex_256_wig, .avx2 }, + .{ .vpmaxsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xee }, 0, .vex_256_wig, .avx2 }, + .{ .vpmaxsd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3d }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmaxub, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xde }, 0, .vex_256_wig, .avx2 }, + .{ .vpmaxuw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3e }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmaxud, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3f }, 0, .vex_256_wig, .avx2 }, + + .{ .vpminsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x38 }, 0, .vex_256_wig, .avx2 }, + .{ .vpminsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xea }, 0, .vex_256_wig, .avx2 }, + .{ .vpminsd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x39 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpminub, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xda }, 0, .vex_256_wig, .avx2 }, + .{ .vpminuw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3a }, 0, .vex_256_wig, .avx2 }, + + .{ .vpminud, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x3b }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmovmskb, .rm, .{ .r32, .ymm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovmskb, .rm, .{ .r64, .ymm }, .{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmovsxbw, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovsxbd, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovsxbq, .rm, .{ .ymm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovsxwd, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovsxwq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovsxdq, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmovzxbw, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovzxbd, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovzxbq, .rm, .{ .ymm, .xmm_m32 }, .{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovzxwd, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovzxwq, .rm, .{ .ymm, .xmm_m64 }, .{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 }, + .{ .vpmovzxdq, .rm, .{ .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmuldq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x28 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmulhrsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0b }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmulhuw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe4 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmulhw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmulld, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmullw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd5 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpmuludq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf4 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpor, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 }, + + .{ .vpsadbw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf6 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpshufb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 }, + .{ .vpshufd, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpshufhw, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpshuflw, .rmi, .{ .ymm, .ymm_m256, .imm8 }, .{ 0xf2, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpsignb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x08 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsignw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x09 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsignd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x0a }, 0, .vex_256_wig, .avx2 }, + + .{ .vpsllw, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf1 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsllw, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 6, .vex_256_wig, .avx2 }, + .{ .vpslld, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf2 }, 0, .vex_256_wig, .avx2 }, + .{ .vpslld, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 6, .vex_256_wig, .avx2 }, + .{ .vpsllq, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xf3 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsllq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 6, .vex_256_wig, .avx2 }, + + .{ .vpslldq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 7, .vex_256_wig, .avx2 }, + + .{ .vpsllvd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_128_w0, .avx2 }, + .{ .vpsllvq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_128_w1, .avx2 }, + .{ .vpsllvd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_256_w0, .avx2 }, + .{ .vpsllvq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x47 }, 0, .vex_256_w1, .avx2 }, + + .{ .vpsraw, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe1 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsraw, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 4, .vex_256_wig, .avx2 }, + .{ .vpsrad, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xe2 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsrad, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 4, .vex_256_wig, .avx2 }, + + .{ .vpsravd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x46 }, 0, .vex_128_w0, .avx2 }, + .{ .vpsravd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x46 }, 0, .vex_256_w0, .avx2 }, + + .{ .vpsrlw, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd1 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsrlw, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x71 }, 2, .vex_256_wig, .avx2 }, + .{ .vpsrld, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd2 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsrld, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x72 }, 2, .vex_256_wig, .avx2 }, + .{ .vpsrlq, .rvm, .{ .ymm, .ymm, .xmm_m128 }, .{ 0x66, 0x0f, 0xd3 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsrlq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 2, .vex_256_wig, .avx2 }, + + .{ .vpsrldq, .vmi, .{ .ymm, .ymm, .imm8 }, .{ 0x66, 0x0f, 0x73 }, 3, .vex_256_wig, .avx2 }, + + .{ .vpsrlvd, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_128_w0, .avx2 }, + .{ .vpsrlvq, .rvm, .{ .xmm, .xmm, .xmm_m128 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_128_w1, .avx2 }, + .{ .vpsrlvd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_256_w0, .avx2 }, + .{ .vpsrlvq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0x45 }, 0, .vex_256_w1, .avx2 }, + + .{ .vpsubb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf8 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsubw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xf9 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsubd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfa }, 0, .vex_256_wig, .avx2 }, + + .{ .vpsubsb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe8 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsubsw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xe9 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpsubq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xfb }, 0, .vex_256_wig, .avx2 }, + + .{ .vpsubusb, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd8 }, 0, .vex_256_wig, .avx2 }, + .{ .vpsubusw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xd9 }, 0, .vex_256_wig, .avx2 }, + + .{ .vpunpckhbw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x68 }, 0, .vex_256_wig, .avx2 }, + .{ .vpunpckhwd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x69 }, 0, .vex_256_wig, .avx2 }, + .{ .vpunpckhdq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6a }, 0, .vex_256_wig, .avx2 }, + .{ .vpunpckhqdq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6d }, 0, .vex_256_wig, .avx2 }, + + .{ .vpunpcklbw, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x60 }, 0, .vex_256_wig, .avx2 }, + .{ .vpunpcklwd, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x61 }, 0, .vex_256_wig, .avx2 }, + .{ .vpunpckldq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x62 }, 0, .vex_256_wig, .avx2 }, + .{ .vpunpcklqdq, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x6c }, 0, .vex_256_wig, .avx2 }, + + .{ .vpxor, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0xef }, 0, .vex_256_wig, .avx2 }, + + // ADX + .{ .adcx, .rm, .{ .r32, .rm32 }, .{ 0x66, 0x0f, 0x38, 0xf6 }, 0, .none, .adx }, + .{ .adcx, .rm, .{ .r64, .rm64 }, .{ 0x66, 0x0f, 0x38, 0xf6 }, 0, .long, .adx }, + + .{ .adox, .rm, .{ .r32, .rm32 }, .{ 0xf3, 0x0f, 0x38, 0xf6 }, 0, .none, .adx }, + .{ .adox, .rm, .{ .r64, .rm64 }, .{ 0xf3, 0x0f, 0x38, 0xf6 }, 0, .long, .adx }, + + // VAES + .{ .vaesdec, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xde }, 0, .vex_256_wig, .vaes }, + + .{ .vaesdeclast, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xdf }, 0, .vex_256_wig, .vaes }, + + .{ .vaesenc, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xdc }, 0, .vex_256_wig, .vaes }, + + .{ .vaesenclast, .rvm, .{ .ymm, .ymm, .ymm_m256 }, .{ 0x66, 0x0f, 0x38, 0xdd }, 0, .vex_256_wig, .vaes }, + + // AESKLE + .{ .aesdec128kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xdd }, 0, .none, .kl }, + + .{ .aesdec256kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xdf }, 0, .none, .kl }, + + .{ .aesenc128kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xdc }, 0, .none, .kl }, + + .{ .aesenc256kl, .rm, .{ .xmm, .m }, .{ 0xf3, 0x0f, 0x38, 0xde }, 0, .none, .kl }, + + .{ .encodekey128, .rm, .{ .r32, .r32 }, .{ 0xf3, 0x0f, 0x38, 0xfa }, 0, .none, .kl }, + + .{ .encodekey256, .rm, .{ .r32, .r32 }, .{ 0xf3, 0x0f, 0x38, 0xfb }, 0, .none, .kl }, + + .{ .loadiwkey, .rm, .{ .xmm, .xmm }, .{ 0xf3, 0x0f, 0x38, 0xdc }, 0, .none, .kl }, + .{ .loadiwkey, .rm, .{ .xmm, .xmm, .eax, .xmm0 }, .{ 0xf3, 0x0f, 0x38, 0xdc }, 0, .none, .kl }, + + // AESKLEWIDE_KL + .{ .aesdecwide128kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 1, .none, .widekl }, + + .{ .aesdecwide256kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 3, .none, .widekl }, + + .{ .aesencwide128kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 0, .none, .widekl }, + + .{ .aesencwide256kl, .m, .{ .m }, .{ 0xf3, 0x0f, 0x38, 0xd8 }, 2, .none, .widekl }, +} +// zig fmt: on diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index 595e3583d5d540bdd0fe0ce1727d0a25eed72f50..8918386b35253a004fc6c267d73a09b41e1b4481 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -102,7 +102,7 @@ const DebugFrame = struct { } + switch (target.cpu.arch) { .x86_64 => len: { dev.check(.x86_64_backend); - const Register = @import("../arch/x86_64/bits.zig").Register; + const Register = @import("../codegen/x86_64/bits.zig").Register; break :len uleb128Bytes(1) + sleb128Bytes(-8) + uleb128Bytes(Register.rip.dwarfNum()) + 1 + uleb128Bytes(Register.rsp.dwarfNum()) + sleb128Bytes(-1) + 1 + uleb128Bytes(1); @@ -2349,7 +2349,7 @@ pub fn init(lf: *link.File, format: DW.Format) Dwarf { .debug_aranges = .{ .section = Section.init }, .debug_frame = .{ .header = if (target.cpu.arch == .x86_64 and target.ofmt == .elf) header: { - const Register = @import("../arch/x86_64/bits.zig").Register; + const Register = @import("../codegen/x86_64/bits.zig").Register; break :header comptime .{ .format = .eh_frame, .code_alignment_factor = 1, @@ -4833,7 +4833,7 @@ fn flushWriterError(dwarf: *Dwarf, pt: Zcu.PerThread) (FlushError || Writer.Erro .eh_frame => switch (target.cpu.arch) { .x86_64 => { dev.check(.x86_64_backend); - const Register = @import("../arch/x86_64/bits.zig").Register; + const Register = @import("../codegen/x86_64/bits.zig").Register; for (dwarf.debug_frame.section.units.items) |*unit| { header_aw.clearRetainingCapacity(); try header_aw.ensureTotalCapacity(unit.header_len); diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index f59d2f4a24dd3531ab67abe2abb3920e8cdbdcc8..8d8bc71a3065562867049783e5ea120332b45d0c 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1473,9 +1473,9 @@ const x86_64 = struct { for (insts) |inst| try inst.encode(&writer, .{}); } - const bits = @import("../../arch/x86_64/bits.zig"); - const encoder = @import("../../arch/x86_64/encoder.zig"); - const Disassembler = @import("../../arch/x86_64/Disassembler.zig"); + const bits = @import("../../codegen/x86_64/bits.zig"); + const encoder = @import("../../codegen/x86_64/encoder.zig"); + const Disassembler = @import("../../codegen/x86_64/Disassembler.zig"); const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; }; diff --git a/src/link/MachO/Atom.zig b/src/link/MachO/Atom.zig index 278d28d270492a247752a0a14e741f17a2455587..a8c1ed843e252775d84d00c1ad230163c7bc72dd 100644 --- a/src/link/MachO/Atom.zig +++ b/src/link/MachO/Atom.zig @@ -901,9 +901,9 @@ const x86_64 = struct { for (insts) |inst| try inst.encode(&stream, .{}); } - const bits = @import("../../arch/x86_64/bits.zig"); - const encoder = @import("../../arch/x86_64/encoder.zig"); - const Disassembler = @import("../../arch/x86_64/Disassembler.zig"); + const bits = @import("../../codegen/x86_64/bits.zig"); + const encoder = @import("../../codegen/x86_64/encoder.zig"); + const Disassembler = @import("../../codegen/x86_64/Disassembler.zig"); const Immediate = bits.Immediate; const Instruction = encoder.Instruction; }; diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index d60b91c30f93695076b8299c7e92d8f734812d32..c0aeb01fc00197c31b8286ca6b305fbe302ffef9 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -29,8 +29,8 @@ const leb = std.leb; const log = std.log.scoped(.link); const mem = std.mem; -const Mir = @import("../arch/wasm/Mir.zig"); -const CodeGen = @import("../arch/wasm/CodeGen.zig"); +const Mir = @import("../codegen/wasm/Mir.zig"); +const CodeGen = @import("../codegen/wasm/CodeGen.zig"); const abi = @import("../codegen/wasm/abi.zig"); const Compilation = @import("../Compilation.zig"); const Dwarf = @import("Dwarf.zig"); diff --git a/src/link/Wasm/Flush.zig b/src/link/Wasm/Flush.zig index f728a8578669fa00c74453dc6b628da3cd77cfaa..b5d93259e2f782d3f29f4858cb92bde1a4483593 100644 --- a/src/link/Wasm/Flush.zig +++ b/src/link/Wasm/Flush.zig @@ -9,7 +9,7 @@ const Alignment = Wasm.Alignment; const String = Wasm.String; const Relocation = Wasm.Relocation; const InternPool = @import("../../InternPool.zig"); -const Mir = @import("../../arch/wasm/Mir.zig"); +const Mir = @import("../../codegen/wasm/Mir.zig"); const build_options = @import("build_options"); diff --git a/src/link/riscv.zig b/src/link/riscv.zig index 85cc491603b00646db523395f227a40a6881bb24..cfc42ac33969e1596982513fbad2216a4c52b62f 100644 --- a/src/link/riscv.zig +++ b/src/link/riscv.zig @@ -113,5 +113,5 @@ pub const Eflags = packed struct(u32) { const mem = std.mem; const std = @import("std"); -const encoding = @import("../arch/riscv64/encoding.zig"); +const encoding = @import("../codegen/riscv64/encoding.zig"); const Instruction = encoding.Instruction;